<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>MC-GPU: MC-GPU_v1.3.cu Source File</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript">
  $(document).ready(initResizable);
</script>


</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">MC-GPU
   
   </div>
   
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="annotated.html"><span>Classes</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
      <li><a href="globals.html"><span>File&#160;Members</span></a></li>
    </ul>
  </div>
</div>
<div id="side-nav" class="ui-resizable side-nav-resizable">
  <div id="nav-tree">
    <div id="nav-tree-contents">
    </div>
  </div>
  <div id="splitbar" style="-moz-user-select:none;" 
       class="ui-resizable-handle">
  </div>
</div>
<script type="text/javascript">
  initNavTree('MC-GPU__v1_83_8cu.html','');
</script>
<div id="doc-content">
<div class="header">
  <div class="headertitle">
<div class="title">MC-GPU_v1.3.cu</div>  </div>
</div><!--header-->
<div class="contents">
<a href="MC-GPU__v1_83_8cu.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 
<a name="l00002"></a>00002 <span class="comment">//   ** CHANGE LIST **  See below \section sec_changes_13 List of changes in code version 1.3</span>
<a name="l00003"></a>00003 
<a name="l00004"></a>00004 <span class="comment">// -- Code upgraded from CUDA 4 to CUDA 5, after cutil_inline.h has been eliminated from the SDK:</span>
<a name="l00005"></a>00005 <span class="comment">//           CUDA 5.0: Using &quot;getLastCudaError&quot; instead of &quot;cutilCheckMsg&quot;</span>
<a name="l00006"></a>00006 <span class="comment">//           CUDA 5.0: Using &quot;checkCudaErrors&quot; instead of &quot;cutilSafeCall&quot; or &quot;cutilCheckError&quot;</span>
<a name="l00007"></a>00007 <span class="comment">//           CUDA 5.0: Using &quot;gpuGetMaxGflopsDeviceId instead of &quot;cutGetMaxGflopsDeviceId&quot;</span>
<a name="l00008"></a>00008 <span class="comment">//           CUDA 5.0: Substitute all the CUDA timer functions (cutResetTimer, cutStartTimer, cutStopTimer, cutGetTimerValue) for standard C clock() calls.</span>
<a name="l00009"></a>00009 
<a name="l00010"></a>00010 <span class="comment"></span>
<a name="l00011"></a>00011 <span class="comment">////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00012"></a>00012 <span class="comment"></span><span class="comment">//</span>
<a name="l00013"></a>00013 <span class="comment">//               ****************************</span>
<a name="l00014"></a>00014 <span class="comment">//               *** MC-GPU , version 1.3 ***</span>
<a name="l00015"></a>00015 <span class="comment">//               ****************************</span>
<a name="l00016"></a>00016 <span class="comment">//</span><span class="comment"></span>
<a name="l00017"></a>00017 <span class="comment">/**</span>
<a name="l00018"></a>00018 <span class="comment"> *      \mainpage MC-GPU v1.3</span>
<a name="l00019"></a>00019 <span class="comment"> * </span>
<a name="l00020"></a>00020 <span class="comment"> * \code   </span>
<a name="l00021"></a>00021 <span class="comment"> * </span>
<a name="l00022"></a>00022 <span class="comment"> *               Andreu Badal, PhD (Andreu.Badal-Soler{at}fda.hhs.gov)</span>
<a name="l00023"></a>00023 <span class="comment"> * </span>
<a name="l00024"></a>00024 <span class="comment"> *                  Division of Imaging and Applied Mathematics</span>
<a name="l00025"></a>00025 <span class="comment"> *                  Office of Science and Engineering Laboratories</span>
<a name="l00026"></a>00026 <span class="comment"> *                  Center for Devices and Radiological Health</span>
<a name="l00027"></a>00027 <span class="comment"> *                  U.S. Food and Drug Administration </span>
<a name="l00028"></a>00028 <span class="comment"> * </span>
<a name="l00029"></a>00029 <span class="comment"> *               Code release date: 2012/12/12</span>
<a name="l00030"></a>00030 <span class="comment"> * </span>
<a name="l00031"></a>00031 <span class="comment"> * </span>
<a name="l00032"></a>00032 <span class="comment"> * </span>
<a name="l00033"></a>00033 <span class="comment"> * \endcode</span>
<a name="l00034"></a>00034 <span class="comment"> * </span>
<a name="l00035"></a>00035 <span class="comment"> *</span>
<a name="l00036"></a>00036 <span class="comment"> * </span>
<a name="l00037"></a>00037 <span class="comment"> *   \b MC-GPU [1-4] is a Monte Carlo simulation code that can generate synthetic radiographic</span>
<a name="l00038"></a>00038 <span class="comment"> *   images and computed tomography (CT) scans of realistic models of the human anatomy using the</span>
<a name="l00039"></a>00039 <span class="comment"> *   computational power of commodity Graphics Processing Unit (GPU) cards.</span>
<a name="l00040"></a>00040 <span class="comment"> *   The code implements a massively multi-threaded Monte Carlo simulation algorithm</span>
<a name="l00041"></a>00041 <span class="comment"> *   for the transport of x rays in a voxelized geometry. The x ray interaction models and material</span>
<a name="l00042"></a>00042 <span class="comment"> *   properties have been adapted from \b PENELOPE \b 2006 [5].</span>
<a name="l00043"></a>00043 <span class="comment"> * </span>
<a name="l00044"></a>00044 <span class="comment"> * </span>
<a name="l00045"></a>00045 <span class="comment"> *   \b MC-GPU was developed using the \b CUDA programming model from \b NVIDIA [6] to achieve </span>
<a name="l00046"></a>00046 <span class="comment"> *   maximum performance on NVIDIA GPUs. The code can also be compiled with a standard C compiler</span>
<a name="l00047"></a>00047 <span class="comment"> *   to be executed in a regular CPU. </span>
<a name="l00048"></a>00048 <span class="comment"> *   In a typical medical imaging simulation, the use of GPU computing with MC-GPU has been shown </span>
<a name="l00049"></a>00049 <span class="comment"> *   to provide a speed up of between 20 and 40 times, compared to the execution on a single CPU core.</span>
<a name="l00050"></a>00050 <span class="comment"> * </span>
<a name="l00051"></a>00051 <span class="comment"> *   The MC-GPU code has been described in different scientific publications [1-4].</span>
<a name="l00052"></a>00052 <span class="comment"> *   The main reference of this work, which the users should cite, is the following [1]:</span>
<a name="l00053"></a>00053 <span class="comment"> * \code</span>
<a name="l00054"></a>00054 <span class="comment"> *  Andreu Badal and Aldo Badano, &quot;Accelerating Monte Carlo simulations of </span>
<a name="l00055"></a>00055 <span class="comment"> *  photon transport in a voxelized geometry using a massively parallel </span>
<a name="l00056"></a>00056 <span class="comment"> *  Graphics Processing Unit&quot;, Medical Physics 36, pp. 4878–4880 (2009)</span>
<a name="l00057"></a>00057 <span class="comment"> * \endcode</span>
<a name="l00058"></a>00058 <span class="comment"> *   The main developer of MC-GPU is \b Andreu \b Badal, working at the U.S. \b Food \b and    </span>
<a name="l00059"></a>00059 <span class="comment"> *   \b Drug \b Administration (Center for Devices and Radiological Health, Office of Science</span>
<a name="l00060"></a>00060 <span class="comment"> *   and Engineering Laboratories, Division of Imaging and Applied Mathematics). </span>
<a name="l00061"></a>00061 <span class="comment"> *   The source code of MC-GPU is free and open software in the public domain, as explained</span>
<a name="l00062"></a>00062 <span class="comment"> *   in the Disclaimer section below. </span>
<a name="l00063"></a>00063 <span class="comment"> *   The source code of MC-GPU and its auxiliary files are distributed from the website: http://code.google.com/. </span>
<a name="l00064"></a>00064 <span class="comment"> *</span>
<a name="l00065"></a>00065 <span class="comment"> *  </span>
<a name="l00066"></a>00066 <span class="comment"> *   This documentation has been automatically generated by \b Doxygen parsing the comments in</span>
<a name="l00067"></a>00067 <span class="comment"> *   the MC-GPU source code.</span>
<a name="l00068"></a>00068 <span class="comment"> *   This code is still in development, please report to the author any issue/bug</span>
<a name="l00069"></a>00069 <span class="comment"> *   that you may encounter. Feel free to suggest improvements to the code too!</span>
<a name="l00070"></a>00070 <span class="comment"> * </span>
<a name="l00071"></a>00071 <span class="comment"> * </span>
<a name="l00072"></a>00072 <span class="comment"> *  </span>
<a name="l00073"></a>00073 <span class="comment"> *    \section  sec_changes  List of modifications in different versions of the code </span>
<a name="l00074"></a>00074 <span class="comment"> * </span>
<a name="l00075"></a>00075 <span class="comment"> *      \subsection  sec_changes_v13  Version 1.3 (release date: 2012/12/12)</span>
<a name="l00076"></a>00076 <span class="comment"> * </span>
<a name="l00077"></a>00077 <span class="comment"> *   - Code upgraded to CUDA 5.0 (not compatible with previous versions of CUDA!).</span>
<a name="l00078"></a>00078 <span class="comment"> *   - Removed limit on the amount of projection images that can be simulated per CT scan (source and </span>
<a name="l00079"></a>00079 <span class="comment"> *     detector parameters now stored in global memory and transferring to shared memory at run time </span>
<a name="l00080"></a>00080 <span class="comment"> *     to avoid using the limited constant memory).</span>
<a name="l00081"></a>00081 <span class="comment"> *   - New material dose tally implemented to estimate the dose deposited in each material independently</span>
<a name="l00082"></a>00082 <span class="comment"> *     of the voxel dose tally (the voxel dose tally measures the dose in each material adding the energy </span>
<a name="l00083"></a>00083 <span class="comment"> *     deposited in each voxel of that material within the defined voxelized region-of-interest).</span>
<a name="l00084"></a>00084 <span class="comment"> *   - Interaction loop re-organized to maximize performance (virtual interactions simulated before real ones). </span>
<a name="l00085"></a>00085 <span class="comment"> *   - Improvements and small corrections in the source sampling and tally routines.</span>
<a name="l00086"></a>00086 <span class="comment"> *   - Allow input of material and voxel geometry files compressed with gzip (zlib library now required for compilation).</span>
<a name="l00087"></a>00087 <span class="comment"> * </span>
<a name="l00088"></a>00088 <span class="comment"> * </span>
<a name="l00089"></a>00089 <span class="comment"> * </span>
<a name="l00090"></a>00090 <span class="comment"> *      \subsection  sec_changes_v12  Version 1.2 (release date: 2011/10/25)</span>
<a name="l00091"></a>00091 <span class="comment"> * </span>
<a name="l00092"></a>00092 <span class="comment"> *   - Implemented a voxel dose tally.</span>
<a name="l00093"></a>00093 <span class="comment"> *   - Polyenergetic source model.</span>
<a name="l00094"></a>00094 <span class="comment"> *   - MPI support for simulating individual projections.</span>
<a name="l00095"></a>00095 <span class="comment"> *   - Simulation by time limit.</span>
<a name="l00096"></a>00096 <span class="comment"> *   - Improved flexibility of the CT trajectories, helical scans.</span>
<a name="l00097"></a>00097 <span class="comment"> * </span>
<a name="l00098"></a>00098 <span class="comment"> *  </span>
<a name="l00099"></a>00099 <span class="comment"> *</span>
<a name="l00100"></a>00100 <span class="comment"> *    \section  sec_disc  Disclaimer</span>
<a name="l00101"></a>00101 <span class="comment"> *</span>
<a name="l00102"></a>00102 <span class="comment"> *   This software and documentation (the &quot;Software&quot;) were developed at the Food and</span>
<a name="l00103"></a>00103 <span class="comment"> *   Drug Administration (FDA) by employees of the Federal Government in the course</span>
<a name="l00104"></a>00104 <span class="comment"> *   of their official duties. Pursuant to Title 17, Section 105 of the United States</span>
<a name="l00105"></a>00105 <span class="comment"> *   Code, this work is not subject to copyright protection and is in the public</span>
<a name="l00106"></a>00106 <span class="comment"> *   domain. Permission is hereby granted, free of charge, to any person obtaining a</span>
<a name="l00107"></a>00107 <span class="comment"> *   copy of the Software, to deal in the Software without restriction, including</span>
<a name="l00108"></a>00108 <span class="comment"> *   without limitation the rights to use, copy, modify, merge, publish, distribute,</span>
<a name="l00109"></a>00109 <span class="comment"> *   sublicense, or sell copies of the Software or derivatives, and to permit persons</span>
<a name="l00110"></a>00110 <span class="comment"> *   to whom the Software is furnished to do so. FDA assumes no responsibility</span>
<a name="l00111"></a>00111 <span class="comment"> *   whatsoever for use by other parties of the Software, its source code,</span>
<a name="l00112"></a>00112 <span class="comment"> *   documentation or compiled executables, and makes no guarantees, expressed or</span>
<a name="l00113"></a>00113 <span class="comment"> *   implied, about its quality, reliability, or any other characteristic. Further,</span>
<a name="l00114"></a>00114 <span class="comment"> *   use of this code in no way implies endorsement by the FDA or confers any</span>
<a name="l00115"></a>00115 <span class="comment"> *   advantage in regulatory decisions.  Although this software can be redistributed</span>
<a name="l00116"></a>00116 <span class="comment"> *   and/or modified freely, we ask that any derivative works bear some notice that</span>
<a name="l00117"></a>00117 <span class="comment"> *   they are derived from it, and any modified versions bear some notice that they</span>
<a name="l00118"></a>00118 <span class="comment"> *   have been modified.</span>
<a name="l00119"></a>00119 <span class="comment"> * </span>
<a name="l00120"></a>00120 <span class="comment"> * </span>
<a name="l00121"></a>00121 <span class="comment"> * </span>
<a name="l00122"></a>00122 <span class="comment"> *     \section sec_Intro Code features</span>
<a name="l00123"></a>00123 <span class="comment"> *  </span>
<a name="l00124"></a>00124 <span class="comment"> *   In this section we provide a brief description of the features of the MC-GPU code. A</span>
<a name="l00125"></a>00125 <span class="comment"> *   more complete description of the code can be found in our published articles.</span>
<a name="l00126"></a>00126 <span class="comment"> *   important information regarding the operation of the code is provided as comments in the  </span>
<a name="l00127"></a>00127 <span class="comment"> *   input files of the sample simulations provided with the MC-GPU package.</span>
<a name="l00128"></a>00128 <span class="comment"> *   Detailed information on each function of the code can be found in the complete Doxygen </span>
<a name="l00129"></a>00129 <span class="comment"> *   documentation of the source code</span>
<a name="l00130"></a>00130 <span class="comment"> * </span>
<a name="l00131"></a>00131 <span class="comment"> *   The basic operation of the code consists in adapting the simulation input file </span>
<a name="l00132"></a>00132 <span class="comment"> *   to describe the location and characteristics of the x ray source, define the CT trajectory</span>
<a name="l00133"></a>00133 <span class="comment"> *   (if any), list the materials to be used in the simulation, define the geometry of</span>
<a name="l00134"></a>00134 <span class="comment"> *   the x ray detector and, finally, specify the voxelized object file to be  </span>
<a name="l00135"></a>00135 <span class="comment"> *   used as the simulation material universe. </span>
<a name="l00136"></a>00136 <span class="comment"> *   In the first line of the input file, the user can fix the total number of x rays that have </span>
<a name="l00137"></a>00137 <span class="comment"> *   to be simulated (&gt; 1e5 histories) or the total simulation time (maximum 1e5 seconds). </span>
<a name="l00138"></a>00138 <span class="comment"> * </span>
<a name="l00139"></a>00139 <span class="comment"> * </span>
<a name="l00140"></a>00140 <span class="comment"> *   The coordinate system of the simulated world is determined by the input voxelized geometry.</span>
<a name="l00141"></a>00141 <span class="comment"> *   The origin of coordinates is assumed to be located at the lower-back corner of the voxelized </span>
<a name="l00142"></a>00142 <span class="comment"> *   volume, and the axis are located on the vertices of the voxelized volume. </span>
<a name="l00143"></a>00143 <span class="comment"> *   This means that the lower-back corner of the first voxel is on the origin and the </span>
<a name="l00144"></a>00144 <span class="comment"> *   following voxels are located along the positive X, Y and Z axis (first quadrant).</span>
<a name="l00145"></a>00145 <span class="comment"> * </span>
<a name="l00146"></a>00146 <span class="comment"> * </span>
<a name="l00147"></a>00147 <span class="comment"> *   To simulate the atomic interactions, MC-GPU uses a database of material properties based on the </span>
<a name="l00148"></a>00148 <span class="comment"> *   database from PENELOPE. A PENELOPE 2006 material file can be converted into an MC-GPU material</span>
<a name="l00149"></a>00149 <span class="comment"> *   file using the auxiliary utility &quot;MC-GPU_create_material_data.f&quot; provided with the MC-GPU </span>
<a name="l00150"></a>00150 <span class="comment"> *   package. Pre-defined material files for a set of materials typically used in medical imaging  </span>
<a name="l00151"></a>00151 <span class="comment"> *   simulations are already provided in the folder &quot;MC-GPU_material_files&quot;.</span>
<a name="l00152"></a>00152 <span class="comment"> * </span>
<a name="l00153"></a>00153 <span class="comment"> * </span>
<a name="l00154"></a>00154 <span class="comment"> *   The code includes two tally options: an \b image \b tally that creates projection x-ray images,</span>
<a name="l00155"></a>00155 <span class="comment"> *   and a radiation \b dose \b tally that estimates the dose deposited inside the patient model.</span>
<a name="l00156"></a>00156 <span class="comment"> *   MC-GPU does not currently simulate the transport of electrons and therefore the dose  </span>
<a name="l00157"></a>00157 <span class="comment"> *   deposition tally (KERMA tally rigorously) will not be accurate for high energies or near </span>
<a name="l00158"></a>00158 <span class="comment"> *   material interfaces and small voxels.</span>
<a name="l00159"></a>00159 <span class="comment"> *   In the image tally the images are formed by counting the energy that enters a user-defined 2D </span>
<a name="l00160"></a>00160 <span class="comment"> *   grid of pixels, which is a simple approximation to a noise-free flat-panel detector with</span>
<a name="l00161"></a>00161 <span class="comment"> *   100% detection efficiency. The pixel values have units of eV/cm^2.</span>
<a name="l00162"></a>00162 <span class="comment"> *   Four different images are reported at the end of the simulation, corresponding</span>
<a name="l00163"></a>00163 <span class="comment"> *   to the signal produced by x rays that did not interact between the source and the detector </span>
<a name="l00164"></a>00164 <span class="comment"> *   (non-scattered), x rays that suffered a single Compton (inelastic) interaction, a single </span>
<a name="l00165"></a>00165 <span class="comment"> *   Rayleigh (elastic) interaction, and multi-scattered x rays.</span>
<a name="l00166"></a>00166 <span class="comment"> *   The dose tally counts the energy deposited by each x ray track inside each voxel of the </span>
<a name="l00167"></a>00167 <span class="comment"> *   geometry, within a user-defined volumetric region-of-interest (ROI). The average dose deposited </span>
<a name="l00168"></a>00168 <span class="comment"> *   inside each voxel and in each material (and the associated statistical uncertainties) are reported </span>
<a name="l00169"></a>00169 <span class="comment"> *   at the end of the simulation.</span>
<a name="l00170"></a>00170 <span class="comment"> * </span>
<a name="l00171"></a>00171 <span class="comment"> * </span>
<a name="l00172"></a>00172 <span class="comment"> *   MC-GPU can simulate a single projection image or a full CT scan.</span>
<a name="l00173"></a>00173 <span class="comment"> *   The CT is simulated generating many projection images around the static </span>
<a name="l00174"></a>00174 <span class="comment"> *   voxelized geometry. Currently, the code is limited to perform a simple</span>
<a name="l00175"></a>00175 <span class="comment"> *   CT trajectory rotating around the Z axis. The user can specify the angular shift and longitudinal</span>
<a name="l00176"></a>00176 <span class="comment"> *   translation (pitch) of the source between each projection and also the distance between the </span>
<a name="l00177"></a>00177 <span class="comment"> *   source and the axis of rotation (the axis is assumed to be parallel to the Z axis). </span>
<a name="l00178"></a>00178 <span class="comment"> *   By now, the code does not simulate some relevant components of a CT scanner such as the </span>
<a name="l00179"></a>00179 <span class="comment"> *   anti-scatter grid, a bow-tie filter or a curved detector (flat-panel detector only).</span>
<a name="l00180"></a>00180 <span class="comment"> * </span>
<a name="l00181"></a>00181 <span class="comment"> * </span>
<a name="l00182"></a>00182 <span class="comment"> *   The x ray source is defined as a point source emitting x rays with an energy randomly sampled </span>
<a name="l00183"></a>00183 <span class="comment"> *   from the user-provided energy spectrum. The polyenergetic spectrum is efficiently sampled </span>
<a name="l00184"></a>00184 <span class="comment"> *   using the Walker aliasing algorithm. The emitted cone beam is computationally</span>
<a name="l00185"></a>00185 <span class="comment"> *   collimated to produce a rectangular field on the detector plane, within the azimuthal and </span>
<a name="l00186"></a>00186 <span class="comment"> *   polar angles specified by the user.</span>
<a name="l00187"></a>00187 <span class="comment"> *   The detector plane is automatically located at the specified distance right in front of the</span>
<a name="l00188"></a>00188 <span class="comment"> *   source focal spot, with the collimated cone beam pointing towards the geometric center of the detector.</span>
<a name="l00189"></a>00189 <span class="comment"> * </span>
<a name="l00190"></a>00190 <span class="comment"> * </span>
<a name="l00191"></a>00191 <span class="comment"> *   In order to optimize the particle tracking algorithm (ray-tracing) and minimize </span>
<a name="l00192"></a>00192 <span class="comment"> *   the accesses to the slow GPU main memory, the photon trajectories across the voxels</span>
<a name="l00193"></a>00193 <span class="comment"> *   are computed using the Woodcock tracking algorithm.</span>
<a name="l00194"></a>00194 <span class="comment"> *   With this technique the photons perceive the geometry as a uniform medium</span>
<a name="l00195"></a>00195 <span class="comment"> *   composed of the material of the most attenuating voxel.</span>
<a name="l00196"></a>00196 <span class="comment"> *   In this way, the voxel boundaries do not have to be explicitly calculated and</span>
<a name="l00197"></a>00197 <span class="comment"> *   multiple voxels can be crossed in a single step.</span>
<a name="l00198"></a>00198 <span class="comment"> *   To keep the simulation unbiased, some of the interactions are considered</span>
<a name="l00199"></a>00199 <span class="comment"> *   &quot;virtual&quot; (i.e., do not change the photon energy or direction of movement),</span>
<a name="l00200"></a>00200 <span class="comment"> *   depending on the x ray energy and the actual material at the interaction site.</span>
<a name="l00201"></a>00201 <span class="comment"> *   In typical medical imaging simulations where the most attenuating material is cortical bone,</span>
<a name="l00202"></a>00202 <span class="comment"> *   the Woodcock tracking algorithm gives an speed up of almost one order of magnitude compared</span>
<a name="l00203"></a>00203 <span class="comment"> *   to computing voxel boundaries all the time. However, if the geometry includes a high</span>
<a name="l00204"></a>00204 <span class="comment"> *   density voxel, such as a metallic implant, the performance of the code can be severely </span>
<a name="l00205"></a>00205 <span class="comment"> *   reduced because a large fraction of the sampled interactions will be virtual.</span>
<a name="l00206"></a>00206 <span class="comment"> *  </span>
<a name="l00207"></a>00207 <span class="comment"> * </span>
<a name="l00208"></a>00208 <span class="comment"> *   The random number generator used in PENELOPE [5], RANECU, is also used in the GPU</span>
<a name="l00209"></a>00209 <span class="comment"> *   program. To ensure that the simulated tracks are not correlated, each thread initializes</span>
<a name="l00210"></a>00210 <span class="comment"> *   the generator to a unique position in the random sequence, far enough from the</span>
<a name="l00211"></a>00211 <span class="comment"> *   other threads, using the algorithm implemented in the seedsMLCG code [7].</span>
<a name="l00212"></a>00212 <span class="comment"> * </span>
<a name="l00213"></a>00213 <span class="comment"> * </span>
<a name="l00214"></a>00214 <span class="comment"> *   In a typical simulation, several thousand threads are launched simultaneously in</span>
<a name="l00215"></a>00215 <span class="comment"> *   the GPU, each one of them simulating a batch of several x ray tracks.</span>
<a name="l00216"></a>00216 <span class="comment"> *   If the code is compiled with MPI support (see below), multiple GPUs can be used in parallel. </span>
<a name="l00217"></a>00217 <span class="comment"> *   The code will perform a short speed test to estimate the relative speed of each GPU used </span>
<a name="l00218"></a>00218 <span class="comment"> *   in the simulation and then distribute the number of particles among the available GPUs correspondingly.</span>
<a name="l00219"></a>00219 <span class="comment"> *   If the user specified a time limit in the simulation, all the GPUs will simulate in parallel </span>
<a name="l00220"></a>00220 <span class="comment"> *   for the allowed time. Since the code is already optimized to scale well in  </span>
<a name="l00221"></a>00221 <span class="comment"> *   thousands of GPU threads, it scales almost linearly with the number of GPUs in most </span>
<a name="l00222"></a>00222 <span class="comment"> *   situations, with only a few seconds of overhead in the initialization of the multiple GPUs </span>
<a name="l00223"></a>00223 <span class="comment"> *   and in the reduction of the final results.</span>
<a name="l00224"></a>00224 <span class="comment"> * </span>
<a name="l00225"></a>00225 <span class="comment"> *  </span>
<a name="l00226"></a>00226 <span class="comment"> *  </span>
<a name="l00227"></a>00227 <span class="comment"> * </span>
<a name="l00228"></a>00228 <span class="comment"> *    \section sec_output Code output</span>
<a name="l00229"></a>00229 <span class="comment"> *</span>
<a name="l00230"></a>00230 <span class="comment"> *   At the end of the simulation the code reports the tallied 3D dose distribution and the </span>
<a name="l00231"></a>00231 <span class="comment"> *   final simulated images in RAW binary form, as 32-bits float values. The image data is provided </span>
<a name="l00232"></a>00232 <span class="comment"> *   as a collection of five consecutive images corresponding to: total image (scatter+primaries), </span>
<a name="l00233"></a>00233 <span class="comment"> *   primary particles, Compton, Rayleigh and multi-scatter. </span>
<a name="l00234"></a>00234 <span class="comment"> *   The dose data is reported as two RAW files with the mean dose and twice the standard deviation</span>
<a name="l00235"></a>00235 <span class="comment"> *   of the dose in each voxel of the geometry respectively, within the input ROI.</span>
<a name="l00236"></a>00236 <span class="comment"> *   The average dose deposited in each material of the geometry is also reported to the standard output. </span>
<a name="l00237"></a>00237 <span class="comment"> *   Organ doses can be obtained by post-processing the output dose file, knowing which voxel </span>
<a name="l00238"></a>00238 <span class="comment"> *   corresponds to each organ.</span>
<a name="l00239"></a>00239 <span class="comment"> *   The pixel and voxel dose data values are stored with the X coordinate incrementing first, the Y </span>
<a name="l00240"></a>00240 <span class="comment"> *   coordinate incrementing second, and the Z coordinate incrementing last.</span>
<a name="l00241"></a>00241 <span class="comment"> *   </span>
<a name="l00242"></a>00242 <span class="comment"> *   The program also reports the simulated images and the dose at the Z plane at the level of the x ray </span>
<a name="l00243"></a>00243 <span class="comment"> *   source as ASCII text files. The ASCII output can be readily visualized with the GNUPLOT scripts </span>
<a name="l00244"></a>00244 <span class="comment"> *   distributed with MC-GPU. The header section at the beginning of these text files provides the </span>
<a name="l00245"></a>00245 <span class="comment"> *   information required to easily read the RAW binary files with IMAGEJ, OCTAVE or other programs. </span>
<a name="l00246"></a>00246 <span class="comment"> *  </span>
<a name="l00247"></a>00247 <span class="comment"> * </span>
<a name="l00248"></a>00248 <span class="comment"> * </span>
<a name="l00249"></a>00249 <span class="comment"> *    \section sec_compilation Code compilation and execution</span>
<a name="l00250"></a>00250 <span class="comment"> *</span>
<a name="l00251"></a>00251 <span class="comment"> *   MC-GPU has been developed and tested only in the Linux operating system.</span>
<a name="l00252"></a>00252 <span class="comment"> *   A Makefile script is provided to compile the MC-GPU code in Linux.</span>
<a name="l00253"></a>00253 <span class="comment"> *   The CUDA libraries and the GNU GCC compiler must be previously installed.</span>
<a name="l00254"></a>00254 <span class="comment"> *   The Makefile may have to be edited to modify the library path.</span>
<a name="l00255"></a>00255 <span class="comment"> *   The code requires the &quot;zlib.h&quot; library to be able to open gzipped input files.</span>
<a name="l00256"></a>00256 <span class="comment"> *</span>
<a name="l00257"></a>00257 <span class="comment"> *</span>
<a name="l00258"></a>00258 <span class="comment"> *   MC-GPU uses CUDA to access NVIDIA GPUs but all the actual computations are coded</span>
<a name="l00259"></a>00259 <span class="comment"> *   in standard C and the CUDA-specific commands are enclosed within preprocessor</span>
<a name="l00260"></a>00260 <span class="comment"> *   &quot;if&quot; statements. Defining the pre-processor variable &quot;USING_CUDA&quot; (i.e., </span>
<a name="l00261"></a>00261 <span class="comment"> *   compiling with &quot;-DUSING_CUDA&quot;) the particle transport routines are compiled to simulate</span>
<a name="l00262"></a>00262 <span class="comment"> *   many x ray histories in parallel in an NVIDIA GPU using CUDA. Otherwise, the code is </span>
<a name="l00263"></a>00263 <span class="comment"> *   sequentially executed in the CPU.</span>
<a name="l00264"></a>00264 <span class="comment"> *   The same coding approach has been used to allow the use of multiple GPUs.</span>
<a name="l00265"></a>00265 <span class="comment"> *   Defining the pre-processor variable &quot;USING_MPI&quot; (i.e., compiling with</span>
<a name="l00266"></a>00266 <span class="comment"> *   &quot;-DUSING_MPI&quot;), Message Passing Interface (MPI) library calls are used to share information </span>
<a name="l00267"></a>00267 <span class="comment"> *   between multiple CPU threads in different computers. </span>
<a name="l00268"></a>00268 <span class="comment"> *   Each MPI thread gets a unique id in the CPU and addresses a unique GPU.</span>
<a name="l00269"></a>00269 <span class="comment"> *   At the end of the simulation the images and doses tallied by the different GPUs are </span>
<a name="l00270"></a>00270 <span class="comment"> *   reduced to form single output file equivalent to a sequential simulation of the same </span>
<a name="l00271"></a>00271 <span class="comment"> *   number of particles.</span>
<a name="l00272"></a>00272 <span class="comment"> * </span>
<a name="l00273"></a>00273 <span class="comment"> *   The code can be easily compiled executing the command &quot;make&quot; or running the provided </span>
<a name="l00274"></a>00274 <span class="comment"> *   &quot;./make.sh&quot; script. </span>
<a name="l00275"></a>00275 <span class="comment"> *   Optionally, the code can be executed from the command line with a command like this </span>
<a name="l00276"></a>00276 <span class="comment"> *   (example using CUDA and MPI, openMPI library in this case):</span>
<a name="l00277"></a>00277 <span class="comment"> * \code </span>
<a name="l00278"></a>00278 <span class="comment"> * nvcc -DUSING_CUDA -DUSING_MPI MC-GPU_v1.3.cu -o MC-GPU_v1.3.x -O3</span>
<a name="l00279"></a>00279 <span class="comment"> *  -use_fast_math -L/usr/lib/ -I. -I/usr/local/cuda/include </span>
<a name="l00280"></a>00280 <span class="comment"> *  -I/usr/local/cuda/samples/common/inc -I/usr/local/cuda/samples/shared/inc/ </span>
<a name="l00281"></a>00281 <span class="comment"> *  -I/usr/include/openmpi  -lmpi -lz --ptxas-options=-v </span>
<a name="l00282"></a>00282 <span class="comment"> *  -gencode=arch=compute_20,code=sm_20 -gencode=arch=compute_30,code=sm_30</span>
<a name="l00283"></a>00283 <span class="comment"> * \endcode</span>
<a name="l00284"></a>00284 <span class="comment"> * </span>
<a name="l00285"></a>00285 <span class="comment"> *   The same source code can also be compiled for a regular CPU using:</span>
<a name="l00286"></a>00286 <span class="comment"> * \code  </span>
<a name="l00287"></a>00287 <span class="comment"> * gcc -x c -O3 MC-GPU_v1.3.cu -o MC-GPU_v1.3_CPU.x -I./ -lm -lz</span>
<a name="l00288"></a>00288 <span class="comment"> * \endcode </span>
<a name="l00289"></a>00289 <span class="comment"> *   </span>
<a name="l00290"></a>00290 <span class="comment"> *   To run a simulation (and keep the information reported to the standard</span>
<a name="l00291"></a>00291 <span class="comment"> *   output in an external file) the compiled code can be executed as:</span>
<a name="l00292"></a>00292 <span class="comment"> * \code </span>
<a name="l00293"></a>00293 <span class="comment"> * ./MC-GPU_v1.3.x MC-GPU_v1.3.in | tee MC-GPU_v1.3.out </span>
<a name="l00294"></a>00294 <span class="comment"> * \endcode </span>
<a name="l00295"></a>00295 <span class="comment"> * </span>
<a name="l00296"></a>00296 <span class="comment"> *   All simulation can be executed in the same way using the code compiled for the CPU </span>
<a name="l00297"></a>00297 <span class="comment"> *   or the GPU (however, the number of histories should be reduced for the CPU to finish </span>
<a name="l00298"></a>00298 <span class="comment"> *   the simulation in a reasonable time).</span>
<a name="l00299"></a>00299 <span class="comment"> *   To run the simulation in parallel with MPI in multiple GPUs (or CPU cores) in the</span>
<a name="l00300"></a>00300 <span class="comment"> *   current computer the user can execute:</span>
<a name="l00301"></a>00301 <span class="comment"> * \code </span>
<a name="l00302"></a>00302 <span class="comment"> * mpirun -n 4 ./MC-GPU_v1.3.x MC-GPU_v1.3.in</span>
<a name="l00303"></a>00303 <span class="comment"> * \endcode  </span>
<a name="l00304"></a>00304 <span class="comment"> * </span>
<a name="l00305"></a>00305 <span class="comment"> *   To use GPUs in different computers, the user must make sure all computers can access the simulation</span>
<a name="l00306"></a>00306 <span class="comment"> *   files and that the libraries are correctly set up in all nodes. </span>
<a name="l00307"></a>00307 <span class="comment"> *   To execute a simulation (with verbose MPI information being reported):</span>
<a name="l00308"></a>00308 <span class="comment"> * \code  </span>
<a name="l00309"></a>00309 <span class="comment"> * mpirun --tag-output -v -x LD_LIBRARY_PATH -hostfile myhostfile.txt -n 8 </span>
<a name="l00310"></a>00310 <span class="comment"> * /fullPath/MC-GPU_v1.3.x /fullPath/MC-GPU_v1.3.in | tee MC-GPU_v1.3.out</span>
<a name="l00311"></a>00311 <span class="comment"> * \endcode  </span>
<a name="l00312"></a>00312 <span class="comment"></span>
<a name="l00313"></a>00313 <span class="comment"> * </span>
<a name="l00314"></a>00314 <span class="comment"> *   The text file &#39;hostfile&#39; lists the IP addresses and number of computing slots (GPUs) of the</span>
<a name="l00315"></a>00315 <span class="comment"> *   computers collaborating in the simulation. This file is not necessary when using multiple</span>
<a name="l00316"></a>00316 <span class="comment"> *   GPUs in a single workstation. When using multiple computers, the simulation files should </span>
<a name="l00317"></a>00317 <span class="comment"> *   be located in a shared drive to make sure every node can access the input data.  </span>
<a name="l00318"></a>00318 <span class="comment"> *   The different workstations must have different host names in order to be differentiated by</span>
<a name="l00319"></a>00319 <span class="comment"> *   the MPI threads. The multiple threads communicate to each other to make sure they don&#39;t  </span>
<a name="l00320"></a>00320 <span class="comment"> *   use the same GPU in the same workstation. </span>
<a name="l00321"></a>00321 <span class="comment"> * </span>
<a name="l00322"></a>00322 <span class="comment"> *  </span>
<a name="l00323"></a>00323 <span class="comment"> *</span>
<a name="l00324"></a>00324 <span class="comment"> *    \section sec_issues Known issues</span>
<a name="l00325"></a>00325 <span class="comment"> * </span>
<a name="l00326"></a>00326 <span class="comment"> *   In extremely long simulations, it is theoretically possible to cause an overflow of the counters </span>
<a name="l00327"></a>00327 <span class="comment"> *   estimating the mean and standard deviation of the material or voxel doses. If this happen, the </span>
<a name="l00328"></a>00328 <span class="comment"> *   results will be incorrect and even negative or nan values can be reported.</span>
<a name="l00329"></a>00329 <span class="comment"> * </span>
<a name="l00330"></a>00330 <span class="comment"> *</span>
<a name="l00331"></a>00331 <span class="comment"> * </span>
<a name="l00332"></a>00332 <span class="comment"> * </span>
<a name="l00333"></a>00333 <span class="comment"> *    \section sec_ref References</span>
<a name="l00334"></a>00334 <span class="comment"> * </span>
<a name="l00335"></a>00335 <span class="comment"> * -# A. Badal and A. Badano, Accelerating Monte Carlo simulations of photon transport in a voxelized geometry using a massively parallel Graphics Processing Unit, Med. Phys. 36, p. 4878-4880 (2009)</span>
<a name="l00336"></a>00336 <span class="comment"> * -# A. Badal and A. Badano, Monte Carlo Simulation of X-Ray Imaging Using a Graphics Processing Unit, IEEE NSC-MIC, Conference Record , HP3–1, p. 4081-4084 (2009)</span>
<a name="l00337"></a>00337 <span class="comment"> * -# A. Badal, I. Kyprianou, D. Sharma and A. Badano, Fast cardiac CT simulation using a Graphics Processing Unit-accelerated Monte Carlo code, Proc. SPIE Medical Imaging Conference 7622, p. 762231 (2010)</span>
<a name="l00338"></a>00338 <span class="comment"> * -# A. Badal and A. Badano, Fast Simulation of Radiographic Images Using a Monte Carlo X-Ray Transport Algorithm Implemented in CUDA, Chapter 50 of GPU Computing Gems (Emerald Edition), p. 813-830, editor Wen-mei W. Hwu, publisher Morgan Kaufmann (Elsevier), Burlington MA, 2010</span>
<a name="l00339"></a>00339 <span class="comment"> * -# F. Salvat, J. M. Fernandez-Varea and J. Sempau, PENELOPE – A code system for Monte Carlo simulation of electron and photon transport, NEA-OECD, Issy-les-Moulineaux, available at www.nea.fr/html/dbprog/peneloperef.html (2006)</span>
<a name="l00340"></a>00340 <span class="comment"> * -# NVIDIA Corporation, NVIDIA CUDA(TM) Programming Guide, Technical Report available at www.nvidia.com/cuda (2011) </span>
<a name="l00341"></a>00341 <span class="comment"> * -# A. Badal and J. Sempau, A package of Linux scripts for the parallelization of Monte Carlo simulations, Comput. Phys. Commun. 175 (6), p. 440-450 (2006) </span>
<a name="l00342"></a>00342 <span class="comment"> * </span>
<a name="l00343"></a>00343 <span class="comment"> * </span>
<a name="l00344"></a>00344 <span class="comment"> * </span>
<a name="l00345"></a>00345 <span class="comment"> *                      @file    MC-GPU_v1.3.cu</span>
<a name="l00346"></a>00346 <span class="comment"> *                      @author  Andreu Badal (Andreu.Badal-Soler@fda.hhs.gov)</span>
<a name="l00347"></a>00347 <span class="comment"> *                      @date    2012/12/12</span>
<a name="l00348"></a>00348 <span class="comment"> *                        -- MC-GPU v.1.3:  2012/12/12</span>
<a name="l00349"></a>00349 <span class="comment"> *                        -- MC-GPU v.1.2:  2011/10/25</span>
<a name="l00350"></a>00350 <span class="comment"> *                        -- MC-GPU v.1.1:  2010/06/25</span>
<a name="l00351"></a>00351 <span class="comment"> *                        -- MC-GPU v.1.0:  2009/03/17</span>
<a name="l00352"></a>00352 <span class="comment"> */</span> <span class="comment"></span>
<a name="l00353"></a>00353 <span class="comment">////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00354"></a>00354 <span class="comment"></span>
<a name="l00355"></a>00355 
<a name="l00356"></a>00356 
<a name="l00357"></a>00357 <span class="comment">// *** Include header file with the structures and functions declarations</span>
<a name="l00358"></a>00358 <span class="preprocessor">#include &lt;<a class="code" href="MC-GPU__v1_83_8h.html" title="Header file containing the declarations for the MC-GPU code.">MC-GPU_v1.3.h</a>&gt;</span>
<a name="l00359"></a>00359 
<a name="l00360"></a>00360 <span class="comment">// *** Include the computing kernel:</span>
<a name="l00361"></a>00361 <span class="preprocessor">#include &lt;<a class="code" href="MC-GPU__kernel__v1_83_8cu.html" title="Definition of the CUDA GPU kernel for the simulation of x ray tracks in a voxelized geometry...">MC-GPU_kernel_v1.3.cu</a>&gt;</span>
<a name="l00362"></a>00362 
<a name="l00363"></a>00363 <span class="comment"></span>
<a name="l00364"></a>00364 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00365"></a>00365 <span class="comment">//!  Main program of MC-GPU: initialize the simulation enviroment, launch the GPU </span>
<a name="l00366"></a>00366 <span class="comment">//!  kernels that perform the x ray transport and report the final results.</span>
<a name="l00367"></a>00367 <span class="comment">//!  This function reads the description of the simulation from an external file</span>
<a name="l00368"></a>00368 <span class="comment">//!  given in the command line. This input file defines the number of particles to</span>
<a name="l00369"></a>00369 <span class="comment">//!  simulate, the characteristics of the x-ray source and the detector, the number</span>
<a name="l00370"></a>00370 <span class="comment">//!  and spacing of the projections (if simulating a CT), the location of the</span>
<a name="l00371"></a>00371 <span class="comment">//!  material files containing the interaction mean free paths, and the location</span>
<a name="l00372"></a>00372 <span class="comment">//!  of the voxelized geometry file.</span>
<a name="l00373"></a>00373 <span class="comment">//!</span>
<a name="l00374"></a>00374 <span class="comment">//!                            @author  Andreu Badal</span>
<a name="l00375"></a>00375 <span class="comment">//!</span>
<a name="l00376"></a>00376 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00377"></a><a class="code" href="MC-GPU__v1_83_8cu.html#a3c04138a5bfe5d72780bb7e82a18e627">00377</a> <span class="comment"></span><span class="keywordtype">int</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a3c04138a5bfe5d72780bb7e82a18e627" title="Main program of MC-GPU: initialize the simulation enviroment, launch the GPU kernels that perform the...">main</a>(<span class="keywordtype">int</span> argc, <span class="keywordtype">char</span> **argv)
<a name="l00378"></a>00378 {
<a name="l00379"></a>00379 
<a name="l00380"></a>00380   <span class="comment">// -- Start time counter:</span>
<a name="l00381"></a>00381   time_t current_time = time(NULL);             <span class="comment">// Get current time (in seconds)  </span>
<a name="l00382"></a>00382   clock_t clock_start, clock_end, clock_start_beginning;  <span class="comment">// (requires standard header &lt;time.h&gt;)</span>
<a name="l00383"></a>00383   clock_start = clock();                        <span class="comment">// Get current clock counter</span>
<a name="l00384"></a>00384   clock_start_beginning = clock_start;
<a name="l00385"></a>00385   
<a name="l00386"></a>00386 <span class="preprocessor">#ifdef USING_MPI</span>
<a name="l00387"></a>00387 <span class="preprocessor"></span><span class="comment">// -- Using MPI to access multiple GPUs to simulate the x-ray projection image:</span>
<a name="l00388"></a>00388   <span class="keywordtype">int</span> myID = -88, numprocs = -99, return_reduce = -1;
<a name="l00389"></a>00389   MPI_Init(&amp;argc, &amp;argv);                       <span class="comment">// Init MPI and get the current thread ID </span>
<a name="l00390"></a>00390   MPI_Comm_rank(MPI_COMM_WORLD, &amp;myID);
<a name="l00391"></a>00391   MPI_Comm_size(MPI_COMM_WORLD, &amp;numprocs);
<a name="l00392"></a>00392   
<a name="l00393"></a>00393   <span class="keywordtype">char</span> MPI_processor_name[81];             
<a name="l00394"></a>00394   <span class="keywordtype">int</span> resultlen = -1;
<a name="l00395"></a>00395   MPI_Get_processor_name(MPI_processor_name, &amp;resultlen);
<a name="l00396"></a>00396     
<a name="l00397"></a>00397   <span class="keywordtype">char</span>* char_time = ctime(&amp;current_time); char_time[19] = <span class="charliteral">&#39;\0&#39;</span>;   <span class="comment">// The time is located betwen the characters 11 and 19.</span>
<a name="l00398"></a>00398   printf(<span class="stringliteral">&quot;          &gt;&gt; MPI run (myId=%d, numprocs=%d) on processor \&quot;%s\&quot; (time: %s) &lt;&lt;\n&quot;</span>, myID, numprocs, MPI_processor_name, &amp;char_time[11]);
<a name="l00399"></a>00399   fflush(stdout);   <span class="comment">// Clear the screen output buffer</span>
<a name="l00400"></a>00400   MPI_Barrier(MPI_COMM_WORLD);   <span class="comment">// Synchronize MPI threads  </span>
<a name="l00401"></a>00401   
<a name="l00402"></a>00402   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;              -- Time spent initializing the MPI world (MPI_Barrier): %.3f s\n&quot;</span>, ((<span class="keywordtype">double</span>)(clock()-clock_start))/CLOCKS_PER_SEC);
<a name="l00403"></a>00403   
<a name="l00404"></a>00404   
<a name="l00405"></a>00405 <span class="preprocessor">#else  </span>
<a name="l00406"></a>00406 <span class="preprocessor"></span>  <span class="keywordtype">int</span> myID = 0, numprocs = 1;   <span class="comment">// Only one CPU thread used when MPI is not activated (multiple projections will be simulated sequentially).</span>
<a name="l00407"></a>00407 <span class="preprocessor">#endif</span>
<a name="l00408"></a>00408 <span class="preprocessor"></span>
<a name="l00409"></a>00409   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l00410"></a>00410   { 
<a name="l00411"></a>00411       printf(<span class="stringliteral">&quot;\n\n     *****************************************************************************\n&quot;</span>);
<a name="l00412"></a>00412       printf(    <span class="stringliteral">&quot;     ***         MC-GPU, version 1.3 (http://code.google.com/p/mcgpu/)         ***\n&quot;</span>);
<a name="l00413"></a>00413       printf(    <span class="stringliteral">&quot;     ***                                                                       ***\n&quot;</span>);
<a name="l00414"></a>00414       printf(    <span class="stringliteral">&quot;     ***  A. Badal and A. Badano, \&quot;Accelerating Monte Carlo simulations of     *** \n&quot;</span>);
<a name="l00415"></a>00415       printf(    <span class="stringliteral">&quot;     ***  photon transport in a voxelized geometry using a massively parallel  *** \n&quot;</span>);
<a name="l00416"></a>00416       printf(    <span class="stringliteral">&quot;     ***  Graphics Processing Unit\&quot;, Medical Physics 36, pp. 4878–4880 (2009)  ***\n&quot;</span>);
<a name="l00417"></a>00417       printf(    <span class="stringliteral">&quot;     ***                                                                       ***\n&quot;</span>);
<a name="l00418"></a>00418       printf(    <span class="stringliteral">&quot;     ***                     Andreu Badal (Andreu.Badal-Soler@fda.hhs.gov)     ***\n&quot;</span>);
<a name="l00419"></a>00419       printf(    <span class="stringliteral">&quot;     *****************************************************************************\n\n&quot;</span>);
<a name="l00420"></a>00420 
<a name="l00421"></a>00421       printf(<span class="stringliteral">&quot;****** Code execution started on: %s\n\n&quot;</span>, ctime(&amp;current_time));  
<a name="l00422"></a>00422       fflush(stdout);
<a name="l00423"></a>00423   }
<a name="l00424"></a>00424     
<a name="l00425"></a>00425   
<a name="l00426"></a>00426 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l00427"></a>00427 <span class="preprocessor"></span>  <span class="comment">// The &quot;MASTER_THREAD&quot; macro prints the messages just once when using MPI threads (it has no effect if MPI is not used):  MASTER_THREAD == &quot;if(0==myID)&quot;</span>
<a name="l00428"></a>00428   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf  (<span class="stringliteral">&quot;\n             *** CUDA SIMULATION IN THE GPU ***\n&quot;</span>);
<a name="l00429"></a>00429 <span class="preprocessor">#else</span>
<a name="l00430"></a>00430 <span class="preprocessor"></span>  <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf  (<span class="stringliteral">&quot;\n             *** SIMULATION IN THE CPU ***\n&quot;</span>);
<a name="l00431"></a>00431 <span class="preprocessor">#endif</span>
<a name="l00432"></a>00432 <span class="preprocessor"></span>
<a name="l00433"></a>00433   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n    -- INITIALIZATION phase:\n&quot;</span>);
<a name="l00434"></a>00434   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> fflush(stdout);   <span class="comment">// Clear the screen output buffer for the master thread</span>
<a name="l00435"></a>00435   
<a name="l00436"></a>00436   <span class="comment"></span>
<a name="l00437"></a>00437 <span class="comment">///////////////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00438"></a>00438 <span class="comment"></span>  
<a name="l00439"></a>00439   
<a name="l00440"></a>00440   <span class="comment">// *** Declare the arrays and structures that will contain the simulation data:</span>
<a name="l00441"></a>00441 
<a name="l00442"></a>00442   <span class="keyword">struct </span><a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a> voxel_data;          <span class="comment">// Define the geometric constants of the voxel file</span>
<a name="l00443"></a>00443   <span class="keyword">struct </span><a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a> detector_data[MAX_NUM_PROJECTIONS];  <span class="comment">// Define an x ray detector (for each projection)</span>
<a name="l00444"></a>00444   <span class="keyword">struct </span><a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a> source_data[MAX_NUM_PROJECTIONS];      <span class="comment">// Define the particles source (for each projection)</span>
<a name="l00445"></a>00445   <span class="keyword">struct </span><a class="code" href="structsource__energy__struct.html" title="Structure storing the source energy spectrum data to be sampled using the Walker aliasing algorithm...">source_energy_struct</a> source_energy_data;    <span class="comment">// Define the source energy spectrum</span>
<a name="l00446"></a>00446   <span class="keyword">struct </span><a class="code" href="structlinear__interp.html" title="Structure with the basic data required by the linear interpolation of the mean free paths: number of ...">linear_interp</a> mfp_table_data;     <span class="comment">// Constant data for the linear interpolation</span>
<a name="l00447"></a>00447   <span class="keyword">struct </span><a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>  compton_table;    <span class="comment">// Structure containing Compton sampling data (to be copied to CONSTANT memory)</span>
<a name="l00448"></a>00448   <span class="keyword">struct </span><a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a> rayleigh_table;   <span class="comment">// Structure containing Rayleigh sampling data (to be copied to CONSTANT memory)</span>
<a name="l00449"></a>00449   
<a name="l00450"></a>00450   <a class="code" href="structfloat2.html">float2</a> *voxel_mat_dens = NULL;           <span class="comment">// Poiter where voxels array will be allocated</span>
<a name="l00451"></a>00451   <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> voxel_mat_dens_bytes = 0;   <span class="comment">// Size (in bytes) of the voxels array (using unsigned int to allocate up to 4.2GBytes)</span>
<a name="l00452"></a>00452   <span class="keywordtype">float</span> density_max[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];
<a name="l00453"></a>00453   <span class="keywordtype">float</span> density_nominal[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];
<a name="l00454"></a>00454   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> *image = NULL;    <span class="comment">// Poiter where image array will be allocated</span>
<a name="l00455"></a>00455   <span class="keywordtype">int</span> image_bytes = -1;                    <span class="comment">// Size of the image array</span>
<a name="l00456"></a>00456   <span class="keywordtype">int</span> mfp_table_bytes = -1, mfp_Woodcock_table_bytes = -1;   <span class="comment">// Size of the table arrays</span>
<a name="l00457"></a>00457   <a class="code" href="structfloat2.html">float2</a> *mfp_Woodcock_table = NULL;                <span class="comment">// Linear interpolation data for the Woodcock mean free path [cm]</span>
<a name="l00458"></a>00458   <a class="code" href="structfloat3.html">float3</a> *mfp_table_a = NULL, *mfp_table_b = NULL;  <span class="comment">// Linear interpolation data for 3 different interactions:</span>
<a name="l00459"></a>00459                                               <span class="comment">//  (1) inverse total mean free path (divided by density, cm^2/g)</span>
<a name="l00460"></a>00460                                               <span class="comment">//  (2) inverse Compton mean free path (divided by density, cm^2/g)</span>
<a name="l00461"></a>00461                                               <span class="comment">//  (3) inverse Rayleigh mean free path (divided by density, cm^2/g)</span>
<a name="l00462"></a>00462   <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_x_min, dose_ROI_x_max, dose_ROI_y_min, dose_ROI_y_max, dose_ROI_z_min, dose_ROI_z_max;  <span class="comment">// Coordinates of the dose region of interest (ROI)</span>
<a name="l00463"></a>00463   <a class="code" href="structulonglong2.html">ulonglong2</a> *voxels_Edep = NULL;           <span class="comment">// Poiter where the voxel energy deposition array will be allocated</span>
<a name="l00464"></a>00464   <span class="keywordtype">int</span> voxels_Edep_bytes = 0;                      <span class="comment">// Size of the voxel Edep array</span>
<a name="l00465"></a>00465   
<a name="l00466"></a>00466   <a class="code" href="structulonglong2.html">ulonglong2</a> materials_dose[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];    <span class="comment">// Array for tally_materials_dose.     !!tally_materials_dose!!</span>
<a name="l00467"></a>00467   <span class="keywordtype">int</span> kk;
<a name="l00468"></a>00468   <span class="keywordflow">for</span>(kk=0;kk&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>;kk++) 
<a name="l00469"></a>00469   {  
<a name="l00470"></a>00470     materials_dose[kk].<a class="code" href="structulonglong2.html#a0d94e96b26765bf28a1a4b76391b88bb">x</a> = 0;       <span class="comment">// Initializing data                  !!tally_materials_dose!!</span>
<a name="l00471"></a>00471     materials_dose[kk].<a class="code" href="structulonglong2.html#a8453eaa1efda22c449296aa2ebcf0c7f">y</a> = 0;
<a name="l00472"></a>00472     density_nominal[kk]  =-1.0f;
<a name="l00473"></a>00473   }
<a name="l00474"></a>00474 
<a name="l00475"></a>00475   clock_t clock_kernel;     <span class="comment">// Using only cpu timers after CUDA 5.0</span>
<a name="l00476"></a>00476 
<a name="l00477"></a>00477   <span class="keywordtype">double</span> time_elapsed_MC_loop = 0.0, time_total_MC_simulation = 0.0, time_total_MC_init_report = 0.0;
<a name="l00478"></a>00478   
<a name="l00479"></a>00479 
<a name="l00480"></a>00480   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories;
<a name="l00481"></a>00481   <span class="keywordtype">int</span> histories_per_thread, seed_input, num_threads_per_block, gpu_id, num_projections;
<a name="l00482"></a>00482   <span class="keywordtype">int</span> flag_material_dose = -2;
<a name="l00483"></a>00483   <span class="keywordtype">double</span> D_angle=-1.0, angularROI_0=0.0, angularROI_1=360.0, initial_angle=0.0, SRotAxisD=-1.0, vertical_translation_per_projection=0.0;
<a name="l00484"></a>00484   <span class="keywordtype">char</span> file_name_voxels[250], file_name_materials[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>][250], file_name_output[250], file_dose_output[250], file_name_espc[250];
<a name="l00485"></a>00485 
<a name="l00486"></a>00486   <span class="comment">// *** Read the input file given in the command line and return the significant data:</span>
<a name="l00487"></a>00487   <a class="code" href="MC-GPU__v1_83_8cu.html#a04125b2129832959744717839c968ebb" title="Read the input file given in the command line and return the significant data.">read_input</a>(argc, argv, myID, &amp;total_histories, &amp;seed_input, &amp;gpu_id, &amp;num_threads_per_block, &amp;histories_per_thread, detector_data, &amp;image, &amp;image_bytes, source_data, &amp;source_energy_data, file_name_voxels, file_name_materials, file_name_output, file_name_espc, &amp;num_projections, &amp;D_angle, &amp;angularROI_0, &amp;angularROI_1, &amp;initial_angle, &amp;voxels_Edep, &amp;voxels_Edep_bytes, file_dose_output, &amp;dose_ROI_x_min, &amp;dose_ROI_x_max, &amp;dose_ROI_y_min, &amp;dose_ROI_y_max, &amp;dose_ROI_z_min, &amp;dose_ROI_z_max, &amp;SRotAxisD, &amp;vertical_translation_per_projection, &amp;flag_material_dose);
<a name="l00488"></a>00488 
<a name="l00489"></a>00489 
<a name="l00490"></a>00490 
<a name="l00491"></a>00491   <span class="comment">// *** Read the energy spectrum and initialize its sampling with the Walker aliasing method:</span>
<a name="l00492"></a>00492   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;    -- Reading the energy spectrum and initializing the Walker aliasing sampling algorithm.\n&quot;</span>);
<a name="l00493"></a>00493   <span class="keywordtype">float</span> mean_energy_spectrum = 0.0f;  
<a name="l00494"></a>00494   <a class="code" href="MC-GPU__v1_83_8cu.html#ac1d6abb26890276f529eed6b7ceafe89" title="Read the energy spectrum file and initialize the Walker aliasing sampling.">init_energy_spectrum</a>(file_name_espc, &amp;source_energy_data, &amp;mean_energy_spectrum);
<a name="l00495"></a>00495   
<a name="l00496"></a>00496   
<a name="l00497"></a>00497   <span class="comment">// *** Output some of the data read to make sure everything was correctly read:</span>
<a name="l00498"></a>00498   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a>
<a name="l00499"></a>00499   {
<a name="l00500"></a>00500         <span class="keywordflow">if</span> (total_histories &lt; (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)(100000))
<a name="l00501"></a>00501           printf(<span class="stringliteral">&quot;                       simulation time = %lld s\n&quot;</span>, total_histories);
<a name="l00502"></a>00502         <span class="keywordflow">else</span>            
<a name="l00503"></a>00503           printf(<span class="stringliteral">&quot;              x-ray tracks to simulate = %lld\n&quot;</span>, total_histories);
<a name="l00504"></a>00504         printf(<span class="stringliteral">&quot;                   initial random seed = %d\n&quot;</span>, seed_input);        
<a name="l00505"></a>00505         printf(<span class="stringliteral">&quot;      azimuthal (phi), polar apertures = %.6f , %.6f degrees\n&quot;</span>, ((<span class="keywordtype">double</span>)source_data[0].D_phi)*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>, 2.0*(90.0 - acos(((<span class="keywordtype">double</span>)source_data[0].cos_theta_low))*RAD2DEG) );
<a name="l00506"></a>00506         printf(<span class="stringliteral">&quot;                   focal spot position = (%f, %f, %f)\n&quot;</span>, source_data[0].position.x, source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l00507"></a>00507         printf(<span class="stringliteral">&quot;                      source direction = (%f, %f, %f)\n&quot;</span>, source_data[0].direction.x, source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l00508"></a>00508         printf(<span class="stringliteral">&quot;                  initial angle from X = %lf\n&quot;</span>, initial_angle*RAD2DEG);
<a name="l00509"></a>00509         printf(<span class="stringliteral">&quot;              source-detector distance = %f cm\n&quot;</span>, detector_data[0].sdd);        
<a name="l00510"></a>00510         printf(<span class="stringliteral">&quot;                       detector center = (%f, %f, %f)\n&quot;</span>, (source_data[0].position.x + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>),  <span class="comment">// Center of the detector straight ahead of the focal spot.</span>
<a name="l00511"></a>00511                                                                           (source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> * detector_data[0].sdd),
<a name="l00512"></a>00512                                                                           (source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> * detector_data[0].sdd));
<a name="l00513"></a>00513         printf(<span class="stringliteral">&quot;           detector low corner (at +Y) = (%f, %f, %f)\n&quot;</span>, detector_data[0].corner_min_rotated_to_Y.x, detector_data[0].corner_min_rotated_to_Y.y, detector_data[0].corner_min_rotated_to_Y.z);        
<a name="l00514"></a>00514         printf(<span class="stringliteral">&quot;                number of pixels image = %dx%d = %d\n&quot;</span>, detector_data[0].num_pixels.x, detector_data[0].num_pixels.y, detector_data[0].total_num_pixels);
<a name="l00515"></a>00515         printf(<span class="stringliteral">&quot;                            pixel size = %.3fx%.3f cm\n&quot;</span>, 1.0f/detector_data[0].inv_pixel_size_X, 1.0f/detector_data[0].inv_pixel_size_Z);
<a name="l00516"></a>00516         printf(<span class="stringliteral">&quot;                 number of projections = %d\n&quot;</span>, num_projections);
<a name="l00517"></a>00517         <span class="keywordflow">if</span> (num_projections!=1)
<a name="l00518"></a>00518         {
<a name="l00519"></a>00519           printf(<span class="stringliteral">&quot;         source-rotation axis-distance = %lf cm\n&quot;</span>, SRotAxisD);          
<a name="l00520"></a>00520           printf(<span class="stringliteral">&quot;             angle between projections = %lf\n&quot;</span>, D_angle*RAD2DEG);
<a name="l00521"></a>00521           printf(<span class="stringliteral">&quot;            angular region of interest = [%lf,%lf] degrees\n&quot;</span>, angularROI_0*RAD2DEG, angularROI_1*RAD2DEG);
<a name="l00522"></a>00522           printf(<span class="stringliteral">&quot;   vertical translation per projection = %lf cm\n&quot;</span>, vertical_translation_per_projection);
<a name="l00523"></a>00523         }
<a name="l00524"></a>00524         printf(<span class="stringliteral">&quot;                      Input voxel file = %s\n&quot;</span>, file_name_voxels);
<a name="l00525"></a>00525         printf(<span class="stringliteral">&quot;                     Output image file = %s\n&quot;</span>, file_name_output);
<a name="l00526"></a>00526 
<a name="l00527"></a>00527         <span class="keywordflow">if</span> (dose_ROI_x_max&gt;-1)
<a name="l00528"></a>00528         {
<a name="l00529"></a>00529           printf(<span class="stringliteral">&quot;                      Output dose file = %s\n&quot;</span>, file_dose_output);
<a name="l00530"></a>00530           printf(<span class="stringliteral">&quot;         Input region of interest dose = X[%d,%d], Y[%d,%d], Z[%d,%d]\n&quot;</span>, dose_ROI_x_min+1, dose_ROI_x_max+1, dose_ROI_y_min+1, dose_ROI_y_max+1, dose_ROI_z_min+1, dose_ROI_z_max+1);   <span class="comment">// Show ROI with index=1 for the first voxel instead of 0.</span>
<a name="l00531"></a>00531         }
<a name="l00532"></a>00532         
<a name="l00533"></a>00533         printf(<span class="stringliteral">&quot;\n                  Energy spectrum file = %s\n&quot;</span>, file_name_espc);      
<a name="l00534"></a>00534         printf(  <span class="stringliteral">&quot;            number of energy bins read = %d\n&quot;</span>, source_energy_data.<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a>);
<a name="l00535"></a>00535         printf(  <span class="stringliteral">&quot;             minimum, maximum energies = %.3f, %.3f keV\n&quot;</span>, 0.001f*source_energy_data.<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[0], 0.001f*source_energy_data.<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[source_energy_data.<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a>]);
<a name="l00536"></a>00536         printf(  <span class="stringliteral">&quot;                  mean energy spectrum = %.3f keV\n\n&quot;</span>, 0.001f*mean_energy_spectrum);
<a name="l00537"></a>00537         
<a name="l00538"></a>00538         fflush(stdout);       
<a name="l00539"></a>00539   }
<a name="l00540"></a>00540   
<a name="l00541"></a>00541  
<a name="l00542"></a>00542 
<a name="l00543"></a>00543   <span class="comment">// *** Set the detectors and sources for the CT trajectory (if needed, ie, for more than one projection):</span>
<a name="l00544"></a>00544   <span class="keywordflow">if</span> (num_projections != 1)
<a name="l00545"></a>00545   {
<a name="l00546"></a>00546     <a class="code" href="MC-GPU__v1_83_8cu.html#a5f62e6c0430761bba20f1e546d396a27" title="Sets the CT trajectory: store in memory the source and detector rotations that are needed to calculat...">set_CT_trajectory</a>(myID, num_projections, D_angle, angularROI_0, angularROI_1, SRotAxisD, source_data, detector_data, vertical_translation_per_projection);
<a name="l00547"></a>00547   }
<a name="l00548"></a>00548   
<a name="l00549"></a>00549   fflush(stdout);
<a name="l00550"></a>00550         
<a name="l00551"></a>00551 
<a name="l00552"></a>00552   <span class="comment">// *** Read the voxel data and allocate the density map matrix. Return the maximum density:</span>
<a name="l00553"></a>00553   <a class="code" href="MC-GPU__v1_83_8cu.html#aed61d205baa31cea69768bbd68c062c6" title="Read the voxel data and allocate the material and density matrix.">load_voxels</a>(myID, file_name_voxels, density_max, &amp;voxel_data, &amp;voxel_mat_dens, &amp;voxel_mat_dens_bytes, &amp;dose_ROI_x_max, &amp;dose_ROI_y_max, &amp;dose_ROI_z_max);
<a name="l00554"></a>00554   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Total CPU memory allocated for voxels vector and data structures = %f Mbytes\n&quot;</span>, (voxel_mat_dens_bytes+image_bytes+<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a>)+<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>)+<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>)+<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structlinear__interp.html" title="Structure with the basic data required by the linear interpolation of the mean free paths: number of ...">linear_interp</a>)+2*mfp_table_bytes+<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a>)+<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>))/(1024.f*1024.f));
<a name="l00555"></a>00555   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> fflush(stdout);
<a name="l00556"></a>00556 
<a name="l00557"></a>00557   <span class="comment">// *** Read the material mean free paths and set the interaction table in a &quot;linear_interp&quot; structure:</span>
<a name="l00558"></a>00558   <a class="code" href="MC-GPU__v1_83_8cu.html#a55f3171a6cdb59fc43e65aa19c676d80" title="Read the material input files and set the mean free paths and the &quot;linear_interp&quot; structures...">load_material</a>(myID, file_name_materials, density_max, density_nominal, &amp;mfp_table_data, &amp;mfp_Woodcock_table, &amp;mfp_Woodcock_table_bytes, &amp;mfp_table_a, &amp;mfp_table_b, &amp;mfp_table_bytes, &amp;rayleigh_table, &amp;compton_table);
<a name="l00559"></a>00559 
<a name="l00560"></a>00560   <span class="comment">// -- Check that the input material tables and the x-ray source are consistent:</span>
<a name="l00561"></a>00561   <span class="keywordflow">if</span> ( (source_energy_data.<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[0] &lt; mfp_table_data.<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>) || (source_energy_data.<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[source_energy_data.<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a>] &gt; (mfp_table_data.<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a> + (mfp_table_data.<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a>-1)/mfp_table_data.<a class="code" href="structlinear__interp.html#a4b743ecf01ca7d075cc6b36b9ce2f0a1">ide</a>)) )
<a name="l00562"></a>00562   {
<a name="l00563"></a>00563     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l00564"></a>00564     {
<a name="l00565"></a>00565       printf(<span class="stringliteral">&quot;\n\n\n !!ERROR!! The input x-ray source energy spectrum minimum (%.3f eV) and maximum (%.3f eV) energy values\n&quot;</span>, source_energy_data.<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[0], source_energy_data.<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[source_energy_data.<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a>]);
<a name="l00566"></a>00566       printf(  <span class="stringliteral">&quot;           are outside the tabulated energy interval for the material properties tables (from %.3f to %.3f eV)!!\n&quot;</span>, mfp_table_data.<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>, (mfp_table_data.<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>+(mfp_table_data.<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a>-1)/mfp_table_data.<a class="code" href="structlinear__interp.html#a4b743ecf01ca7d075cc6b36b9ce2f0a1">ide</a>));
<a name="l00567"></a>00567       printf(  <span class="stringliteral">&quot;           Please, modify the input energy spectra to fit the tabulated limits or create new tables.\n\n&quot;</span>);
<a name="l00568"></a>00568     }
<a name="l00569"></a>00569 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l00570"></a>00570 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l00571"></a>00571 <span class="preprocessor">    #endif</span>
<a name="l00572"></a>00572 <span class="preprocessor"></span>    exit(-1);
<a name="l00573"></a>00573   }
<a name="l00574"></a>00574 
<a name="l00575"></a>00575   <span class="comment">// -- Pre-compute the total mass of each material present in the voxel phantom (to be used in &quot;report_materials_dose&quot;):</span>
<a name="l00576"></a>00576   <span class="keywordtype">double</span> voxel_volume = 1.0 / ( ((double)voxel_data.<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>) * ((double)voxel_data.<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>) * ((double)voxel_data.<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>) );
<a name="l00577"></a>00577   <span class="keywordtype">double</span> mass_materials[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];
<a name="l00578"></a>00578   <span class="keywordflow">for</span>(kk=0; kk&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; kk++)
<a name="l00579"></a>00579     mass_materials[kk] = 0.0;
<a name="l00580"></a>00580   <span class="keywordflow">for</span>(kk=0; kk&lt;(voxel_data.<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>*voxel_data.<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>*voxel_data.<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>); kk++)  <span class="comment">// For each voxel in the geometry</span>
<a name="l00581"></a>00581     mass_materials[((<span class="keywordtype">int</span>)voxel_mat_dens[kk].x)-1] += ((<span class="keywordtype">double</span>)voxel_mat_dens[kk].y)*voxel_volume;        <span class="comment">// Add material mass = density*volume</span>
<a name="l00582"></a>00582 
<a name="l00583"></a>00583 
<a name="l00584"></a>00584 
<a name="l00585"></a>00585   <span class="comment">// *** Initialize the GPU using the NVIDIA CUDA libraries, if USING_CUDA parameter defined at compile time:</span>
<a name="l00586"></a>00586 <span class="preprocessor">#ifdef USING_CUDA    </span>
<a name="l00587"></a>00587 <span class="preprocessor"></span>  <span class="comment">// -- Declare the pointers to the device global memory, when using the GPU:</span>
<a name="l00588"></a>00588   <a class="code" href="structfloat2.html">float2</a> *voxel_mat_dens_device     = NULL,
<a name="l00589"></a>00589          *mfp_Woodcock_table_device = NULL;
<a name="l00590"></a>00590   <a class="code" href="structfloat3.html">float3</a> *mfp_table_a_device        = NULL,
<a name="l00591"></a>00591          *mfp_table_b_device        = NULL;
<a name="l00592"></a>00592   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> *image_device          = NULL;
<a name="l00593"></a>00593   <span class="keyword">struct </span><a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a> *rayleigh_table_device = NULL;
<a name="l00594"></a>00594   <span class="keyword">struct </span><a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>  *compton_table_device  = NULL;
<a name="l00595"></a>00595   <a class="code" href="structulonglong2.html">ulonglong2</a> *voxels_Edep_device                = NULL;
<a name="l00596"></a>00596   <span class="keyword">struct </span><a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a> *detector_data_device  = NULL;
<a name="l00597"></a>00597   <span class="keyword">struct </span><a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>   *source_data_device    = NULL;  
<a name="l00598"></a>00598   <a class="code" href="structulonglong2.html">ulonglong2</a> *materials_dose_device = NULL;     <span class="comment">// !!tally_materials_dose!!</span>
<a name="l00599"></a>00599 
<a name="l00600"></a>00600   <span class="comment">// -- Sets the CUDA enabled GPU that will be used in the simulation, and allocate and copies the simulation data in the GPU global and constant memories.</span>
<a name="l00601"></a>00601   init_CUDA_device(&amp;gpu_id, myID, numprocs, &amp;voxel_data, source_data, &amp;source_energy_data, detector_data, &amp;mfp_table_data,  <span class="comment">/*Variables GPU constant memory*/</span>
<a name="l00602"></a>00602         voxel_mat_dens, &amp;voxel_mat_dens_device, voxel_mat_dens_bytes,                          <span class="comment">/*Variables GPU global memory*/</span>
<a name="l00603"></a>00603         image, &amp;image_device, image_bytes,
<a name="l00604"></a>00604         mfp_Woodcock_table, &amp;mfp_Woodcock_table_device, mfp_Woodcock_table_bytes,
<a name="l00605"></a>00605         mfp_table_a, mfp_table_b, &amp;mfp_table_a_device, &amp;mfp_table_b_device, mfp_table_bytes,
<a name="l00606"></a>00606         &amp;rayleigh_table, &amp;rayleigh_table_device,
<a name="l00607"></a>00607         &amp;compton_table, &amp;compton_table_device, &amp;detector_data_device, &amp;source_data_device,
<a name="l00608"></a>00608         voxels_Edep, &amp;voxels_Edep_device, voxels_Edep_bytes, &amp;dose_ROI_x_min, &amp;dose_ROI_x_max, &amp;dose_ROI_y_min, &amp;dose_ROI_y_max, &amp;dose_ROI_z_min, &amp;dose_ROI_z_max,
<a name="l00609"></a>00609         materials_dose, &amp;materials_dose_device, flag_material_dose, num_projections);
<a name="l00610"></a>00610 
<a name="l00611"></a>00611   <span class="comment">// -- Constant data already moved to the GPU: clean up unnecessary RAM memory</span>
<a name="l00612"></a>00612   free(mfp_Woodcock_table);
<a name="l00613"></a>00613   free(mfp_table_a);
<a name="l00614"></a>00614   free(mfp_table_b);
<a name="l00615"></a>00615   <span class="keywordflow">if</span> (0!=myID)    <span class="comment">// Keep the geometry data for the MPI root because the voxel densities are still needed to compute the final doses</span>
<a name="l00616"></a>00616     free(voxel_mat_dens);
<a name="l00617"></a>00617     
<a name="l00618"></a>00618 
<a name="l00619"></a>00619 <span class="preprocessor">#endif</span>
<a name="l00620"></a>00620 <span class="preprocessor"></span>  
<a name="l00621"></a>00621   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a>
<a name="l00622"></a>00622   {
<a name="l00623"></a>00623     current_time=time(NULL);
<a name="l00624"></a>00624     printf(<span class="stringliteral">&quot;\n    -- INITIALIZATION finished: elapsed time = %.3f s. \n\n&quot;</span>, ((<span class="keywordtype">double</span>)(clock()-clock_start))/CLOCKS_PER_SEC);
<a name="l00625"></a>00625   }
<a name="l00626"></a>00626   
<a name="l00627"></a>00627 
<a name="l00628"></a>00628 <span class="preprocessor">#ifdef USING_MPI</span>
<a name="l00629"></a>00629 <span class="preprocessor"></span>  fflush(stdout);
<a name="l00630"></a>00630   MPI_Barrier(MPI_COMM_WORLD);   <span class="comment">// Synchronize MPI threads before starting the MC phase.</span>
<a name="l00631"></a>00631 <span class="preprocessor">#endif</span>
<a name="l00632"></a>00632 <span class="preprocessor"></span>
<a name="l00633"></a>00633   <span class="comment"></span>
<a name="l00634"></a>00634 <span class="comment">///////////////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00635"></a>00635 <span class="comment"></span>  
<a name="l00636"></a>00636   
<a name="l00637"></a>00637   
<a name="l00638"></a>00638   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a>
<a name="l00639"></a>00639   {
<a name="l00640"></a>00640     current_time=time(NULL);
<a name="l00641"></a>00641     printf(<span class="stringliteral">&quot;\n\n    -- MONTE CARLO LOOP phase. Time: %s\n\n&quot;</span>, ctime(&amp;current_time)); 
<a name="l00642"></a>00642     fflush(stdout);    
<a name="l00643"></a>00643   }
<a name="l00644"></a>00644 
<a name="l00645"></a>00645   
<a name="l00646"></a>00646   <span class="comment">// -- A number of histories smaller than 24 hours in sec (3600*24=86400) means that the user wants to simulate for the input number of seconds in each GPU, not a fix number of histories:</span>
<a name="l00647"></a>00647   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories_INPUT = total_histories;    <span class="comment">// Save the original input values to be re-used for multiple projections</span>
<a name="l00648"></a>00648   <span class="keywordtype">int</span> seed_input_INPUT = seed_input, doing_speed_test = -1;  
<a name="l00649"></a>00649   <span class="keywordtype">int</span> simulating_by_time = 0;  <span class="comment">// 0==false</span>
<a name="l00650"></a>00650   <span class="keywordflow">if</span> (total_histories &lt; (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)(95000))
<a name="l00651"></a>00651     simulating_by_time = 1;    <span class="comment">// 1=true</span>
<a name="l00652"></a>00652       
<a name="l00653"></a>00653 
<a name="l00654"></a>00654 
<a name="l00655"></a>00655   <span class="keywordtype">int</span> num_blocks_speed_test = 0;
<a name="l00656"></a>00656   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> histories_speed_test = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)0, total_histories_speed_test = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)0;
<a name="l00657"></a>00657   <span class="keywordtype">float</span> node_speed = -1.0f, total_speed = 1.0f;
<a name="l00658"></a>00658   <span class="keywordtype">double</span> current_angle;
<a name="l00659"></a>00659   <span class="keywordtype">int</span> num_p;  <span class="comment">// == current projection number</span>
<a name="l00660"></a>00660   
<a name="l00661"></a>00661    <span class="comment">// *** CT simulation: find the current projection angle and start Monte Carlo simulation: </span>
<a name="l00662"></a>00662    
<a name="l00663"></a>00663   <span class="keywordflow">for</span> (num_p=0; num_p&lt;num_projections; num_p++)
<a name="l00664"></a>00664   {
<a name="l00665"></a>00665     
<a name="l00666"></a>00666     <span class="comment">// -- Check if this projection is inside the input angular region of interest (the angle can be negative, or larger than 360 in helical scans):</span>
<a name="l00667"></a>00667     current_angle = initial_angle + num_p * D_angle;       
<a name="l00668"></a>00668     
<a name="l00669"></a>00669     <span class="keywordflow">if</span> ((current_angle &lt; angularROI_0) || (current_angle &gt; angularROI_1))
<a name="l00670"></a>00670     {
<a name="l00671"></a>00671       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;         &lt;&lt; Skipping projection #%d of %d &gt;&gt; Angle %f degrees: outside angular region of interest.\n&quot;</span>, num_p+1, num_projections, current_angle*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>);
<a name="l00672"></a>00672       <span class="keywordflow">continue</span>;   <span class="comment">// Cycle loop: do not simulate this projection!</span>
<a name="l00673"></a>00673     }
<a name="l00674"></a>00674       
<a name="l00675"></a>00675     <span class="keywordflow">if</span> (num_projections!=1)
<a name="l00676"></a>00676       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n\n   &lt;&lt; Simulating Projection %d of %d &gt;&gt; Angle: %lf degrees.\n\n\n&quot;</span>, num_p+1, num_projections, current_angle*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>);          
<a name="l00677"></a>00677 
<a name="l00678"></a>00678     
<a name="l00679"></a>00679     clock_start = clock();   <span class="comment">// Start the CPU clock</span>
<a name="l00680"></a>00680     
<a name="l00681"></a>00681 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l00682"></a>00682 <span class="preprocessor"></span>      
<a name="l00683"></a>00683     <span class="comment">// *** Simulate in the GPUs the input amount of time or amount of particles:</span>
<a name="l00684"></a>00684     
<a name="l00685"></a>00685     <span class="comment">// -- Estimate GPU speed to use a total simulation time or multiple GPUs:    </span>
<a name="l00686"></a>00686     
<a name="l00687"></a>00687     <span class="keywordflow">if</span> ( simulating_by_time==0 &amp;&amp;   <span class="comment">// Simulating a fixed number of particles, not a fixed time (so performing the speed test only once)</span>
<a name="l00688"></a>00688          node_speed&gt;0.0f &amp;&amp;         <span class="comment">// Speed test already performed for a previous projection in this simulation (node_speed and total_speed variables set)</span>
<a name="l00689"></a>00689          numprocs&gt;1)                <span class="comment">// Using multiple GPUs (ie, multiple MPI threads)</span>
<a name="l00690"></a>00690     { 
<a name="l00691"></a>00691       <span class="comment">// -- Simulating successive projections after the first one with a fix number of particles, with multiple MPI threads: re-use the speed test results from the first projection image:</span>
<a name="l00692"></a>00692       total_histories = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)(0.5 + ((<span class="keywordtype">double</span>)total_histories_INPUT) * (((double)node_speed)/total_speed));  
<a name="l00693"></a>00693       doing_speed_test = 0;   <span class="comment">// No speed test for this projection.</span>
<a name="l00694"></a>00694     }
<a name="l00695"></a>00695     <span class="keywordflow">else</span> <span class="keywordflow">if</span> ( simulating_by_time==1 || numprocs&gt;1)
<a name="l00696"></a>00696     {
<a name="l00697"></a>00697       <span class="comment">// -- Simulating with a time limit OR multiple MPI threads for the first time (num_p==0): run a speed test to calculate the speed of the current GPU and distribute the number of particles to the multiple GPUs or estimate the total number of particles required to run the input amount of time:      </span>
<a name="l00698"></a>00698       <span class="comment">//    Note that this ELSE IF block will be skipped if we are using a single MPI thread and a fix number of particles.</span>
<a name="l00699"></a>00699 
<a name="l00700"></a>00700       doing_speed_test = 1;   <span class="comment">// Remember that we are performing the speed test to make sure we add the test histories to the total before the tally reports.</span>
<a name="l00701"></a>00701 
<a name="l00702"></a>00702       <span class="keywordflow">if</span> (node_speed&lt;0.0f)    <span class="comment">// Speed test not performed before (first projection being simulated): set num_blocks_speed_test and histories_speed_test.</span>
<a name="l00703"></a>00703       {
<a name="l00704"></a>00704         num_blocks_speed_test = guestimate_GPU_performance(gpu_id);  <span class="comment">// Guestimating a good number of blocks to estimate the speed of different generations of GPUs. Slower GPUs will simulate less particles and hopefully the fastest GPUs will not have to wait much.</span>
<a name="l00705"></a>00705         
<a name="l00706"></a>00706               <span class="comment">// !!DeBuG!! Error in code version 1.2 has been corrected here. Old code:   histories_speed_test = (unsigned long long int)(num_blocks_speed_test*num_threads_per_block)*(unsigned long long int)(histories_per_thread);</span>
<a name="l00707"></a>00707         
<a name="l00708"></a>00708       }
<a name="l00709"></a>00709       
<a name="l00710"></a>00710       histories_speed_test = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)(num_blocks_speed_test*num_threads_per_block)*(<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)(histories_per_thread);
<a name="l00711"></a>00711 
<a name="l00712"></a>00712 
<a name="l00713"></a>00713       <span class="comment">// Re-load the input total number of histories and the random seed:</span>
<a name="l00714"></a>00714       total_histories = total_histories_INPUT;
<a name="l00715"></a>00715       seed_input = seed_input_INPUT;                
<a name="l00716"></a>00716       
<a name="l00717"></a>00717       dim3  blocks_speed_test(num_blocks_speed_test, 1);
<a name="l00718"></a>00718       dim3 threads_speed_test(num_threads_per_block, 1);
<a name="l00719"></a>00719 
<a name="l00720"></a>00720       
<a name="l00721"></a>00721       <span class="comment">// -- Init the current random number generator seed to avoid overlapping sequences with other MPI threads:      </span>
<a name="l00722"></a>00722       <span class="keywordflow">if</span> (simulating_by_time == 1) 
<a name="l00723"></a>00723         <span class="comment">// Simulating by time: set an arbitrary huge number of particles to skip.</span>
<a name="l00724"></a>00724         <a class="code" href="MC-GPU__v1_83_8cu.html#a6577c51b64b94b13c4e95df776275b5a" title="Initialize the first seed of the pseudo-random number generator (PRNG) RANECU to a position far away ...">update_seed_PRNG</a>( (myID + num_p*numprocs), (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)(123456789012), &amp;seed_input);     <span class="comment">// Set the random number seed far from any other MPI thread (myID) and away from the seeds used in the previous projections (num_p*numprocs).</span>
<a name="l00725"></a>00725       <span class="keywordflow">else</span>  
<a name="l00726"></a>00726         <span class="comment">// Simulating by histories</span>
<a name="l00727"></a>00727         <a class="code" href="MC-GPU__v1_83_8cu.html#a6577c51b64b94b13c4e95df776275b5a" title="Initialize the first seed of the pseudo-random number generator (PRNG) RANECU to a position far away ...">update_seed_PRNG</a>( (myID + num_p*numprocs), total_histories, &amp;seed_input);   <span class="comment">//  Using different random seeds for each projection            </span>
<a name="l00728"></a>00728       
<a name="l00729"></a>00729 <span class="preprocessor">      #ifdef USING_MPI</span>
<a name="l00730"></a>00730 <span class="preprocessor"></span>        printf(<span class="stringliteral">&quot;        ==&gt; CUDA (MPI process #%d in \&quot;%s\&quot;): estimate GPU speed executing %d blocks of %d threads, %d histories per thread: %lld histories in total (random seed: %d).\n&quot;</span>, myID, MPI_processor_name, num_blocks_speed_test, num_threads_per_block, histories_per_thread, histories_speed_test, seed_input);
<a name="l00731"></a>00731 <span class="preprocessor">      #else</span>
<a name="l00732"></a>00732 <span class="preprocessor"></span>        printf(<span class="stringliteral">&quot;        ==&gt; CUDA: Estimating the GPU speed executing %d blocks of %d threads, %d histories per thread: %lld histories in total.\n&quot;</span>, num_blocks_speed_test, num_threads_per_block, histories_per_thread, histories_speed_test);
<a name="l00733"></a>00733 <span class="preprocessor">      #endif  </span>
<a name="l00734"></a>00734 <span class="preprocessor"></span>      fflush(stdout); 
<a name="l00735"></a>00735       
<a name="l00736"></a>00736       clock_kernel = clock();
<a name="l00737"></a>00737       
<a name="l00738"></a>00738       <span class="comment">// -- Launch Monte Carlo simulation kernel for the speed test:</span>
<a name="l00739"></a>00739       track_particles&lt;&lt;&lt;blocks_speed_test,threads_speed_test&gt;&gt;&gt;(histories_per_thread, num_p, seed_input, image_device, voxels_Edep_device, voxel_mat_dens_device, mfp_Woodcock_table_device, mfp_table_a_device, mfp_table_b_device, rayleigh_table_device, compton_table_device, detector_data_device, source_data_device, materials_dose_device);
<a name="l00740"></a>00740       
<a name="l00741"></a>00741       
<a name="l00742"></a>00742 <span class="preprocessor">      #ifdef USING_MPI    </span>
<a name="l00743"></a>00743 <span class="preprocessor"></span>        <span class="comment">// Find out the total number of histories simulated in the speed test by all the GPUs. Note that this MPI call will be executed in parallel with the GPU kernel because it is located before the cudaThreadSynchronize command!</span>
<a name="l00744"></a>00744       
<a name="l00745"></a>00745         return_reduce = MPI_Allreduce(&amp;histories_speed_test, &amp;total_histories_speed_test, 1, MPI_UNSIGNED_LONG, MPI_SUM, MPI_COMM_WORLD);  
<a name="l00746"></a>00746         <span class="keywordflow">if</span> (MPI_SUCCESS != return_reduce)
<a name="l00747"></a>00747           printf(<span class="stringliteral">&quot;\n\n !!ERROR!! Error reducing (MPI_Allreduce) the total number of histories in the speed test test??? return_reduce = %d for thread %d\n\n\n&quot;</span>, return_reduce, myID);
<a name="l00748"></a>00748         <span class="keywordflow">else</span>
<a name="l00749"></a>00749 <span class="preprocessor">      #else</span>
<a name="l00750"></a>00750 <span class="preprocessor"></span>        total_histories_speed_test = histories_speed_test;
<a name="l00751"></a>00751 <span class="preprocessor">      #endif</span>
<a name="l00752"></a>00752 <span class="preprocessor"></span>            
<a name="l00753"></a>00753       cudaThreadSynchronize();    <span class="comment">// Force the runtime to wait until GPU kernel has completed</span>
<a name="l00754"></a>00754       getLastCudaError(<span class="stringliteral">&quot;\n\n !!Kernel execution failed while simulating particle tracks!! &quot;</span>);   <span class="comment">// Check if the CUDA function returned any error</span>
<a name="l00755"></a>00755 
<a name="l00756"></a>00756       <span class="keywordtype">float</span> speed_test_time = float(clock()-clock_kernel)/CLOCKS_PER_SEC;
<a name="l00757"></a>00757 
<a name="l00758"></a>00758       node_speed = (float) (((<span class="keywordtype">double</span>)histories_speed_test)/speed_test_time); 
<a name="l00759"></a>00759       
<a name="l00760"></a>00760 <span class="preprocessor">      #ifdef USING_MPI  </span>
<a name="l00761"></a>00761 <span class="preprocessor"></span>        printf(<span class="stringliteral">&quot;                 (MPI process #%d): Estimated GPU speed = %lld hist / %.4f s = %.3f hist/s\n&quot;</span>, myID, histories_speed_test, speed_test_time, node_speed);      
<a name="l00762"></a>00762 <span class="preprocessor">      #else</span>
<a name="l00763"></a>00763 <span class="preprocessor"></span>        printf(<span class="stringliteral">&quot;                  Estimated GPU speed = %lld hist / %.3f s = %.3f hist/s\n&quot;</span>, histories_speed_test, speed_test_time, node_speed);        
<a name="l00764"></a>00764 <span class="preprocessor">      #endif</span>
<a name="l00765"></a>00765 <span class="preprocessor"></span>
<a name="l00766"></a>00766       
<a name="l00767"></a>00767       <span class="comment">// -- Init random number generator seed to avoid repeating the random numbers used in the speed test:</span>
<a name="l00768"></a>00768       <a class="code" href="MC-GPU__v1_83_8cu.html#a6577c51b64b94b13c4e95df776275b5a" title="Initialize the first seed of the pseudo-random number generator (PRNG) RANECU to a position far away ...">update_seed_PRNG</a>(1, histories_speed_test, &amp;seed_input);
<a name="l00769"></a>00769       
<a name="l00770"></a>00770       <span class="keywordflow">if</span> (simulating_by_time==1)
<a name="l00771"></a>00771       {
<a name="l00772"></a>00772         <span class="comment">// -- Set number of histories for each GPU when simulating by time:</span>
<a name="l00773"></a>00773         <span class="keywordflow">if</span> (total_histories &gt; speed_test_time)
<a name="l00774"></a>00774           total_histories = (total_histories - speed_test_time)*node_speed;    <span class="comment">// Calculate the total number of remaining histories by &quot;GPU speed&quot; * &quot;remaining time&quot;</span>
<a name="l00775"></a>00775         <span class="keywordflow">else</span>
<a name="l00776"></a>00776           total_histories = 1;       <span class="comment">// Enough particles simulated already, simulate just one more history (block) and report (kernel call would fail if total_histories &lt; or == 0).</span>
<a name="l00777"></a>00777       }
<a name="l00778"></a>00778       <span class="keywordflow">else</span>
<a name="l00779"></a>00779       {
<a name="l00780"></a>00780         
<a name="l00781"></a>00781 <span class="preprocessor">        #ifdef USING_MPI </span>
<a name="l00782"></a>00782 <span class="preprocessor"></span>          <span class="comment">// -- Simulating a fix number of histories divided between all GPUs (execution time variable):                     </span>
<a name="l00783"></a>00783           <span class="comment">//    Compute the fraction of the total speed that accounts for the current MPI thread:</span>
<a name="l00784"></a>00784           return_reduce = MPI_Allreduce(&amp;node_speed, &amp;total_speed, 1, MPI_FLOAT, MPI_SUM, MPI_COMM_WORLD);  <span class="comment">// Sum all the times and send result to all processes</span>
<a name="l00785"></a>00785           
<a name="l00786"></a>00786           <span class="keywordflow">if</span> (MPI_SUCCESS != return_reduce)
<a name="l00787"></a>00787             printf(<span class="stringliteral">&quot;\n\n !!ERROR!! Error reducing (MPI_Allreduce) the speed test results??? return_reduce = %d for thread %d\n\n\n&quot;</span>, return_reduce, myID);
<a name="l00788"></a>00788           <span class="keywordflow">else</span>
<a name="l00789"></a>00789             <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l00790"></a>00790             {
<a name="l00791"></a>00791               printf(<span class="stringliteral">&quot;       -- Total speed for all GPUs (MPI_Allreduce) = %.3f hist/s; total histories simulated in the speed test (MPI_Allreduce) = %lld.\n&quot;</span>, total_speed, total_histories_speed_test);
<a name="l00792"></a>00792               printf(<span class="stringliteral">&quot;          The master thread will simulate %.2f%% of the x rays in the simulation.\n&quot;</span>,  100.0f*node_speed/total_speed);
<a name="l00793"></a>00793             }
<a name="l00794"></a>00794 <span class="preprocessor">        #else</span>
<a name="l00795"></a>00795 <span class="preprocessor"></span>          total_speed = node_speed;
<a name="l00796"></a>00796 <span class="preprocessor">        #endif</span>
<a name="l00797"></a>00797 <span class="preprocessor"></span>
<a name="l00798"></a>00798         <span class="comment">// - Divide the remaining histories among the MPI threads (GPUs) according to their fraction of the total speed (rounding up).</span>
<a name="l00799"></a>00799         <span class="keywordflow">if</span> (total_histories_speed_test &lt; total_histories)
<a name="l00800"></a>00800           total_histories = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)(0.5 + ((<span class="keywordtype">double</span>)(total_histories-total_histories_speed_test)) * ((double)(node_speed/total_speed)));
<a name="l00801"></a>00801         <span class="keywordflow">else</span>
<a name="l00802"></a>00802           total_histories = numprocs;       <span class="comment">// Enough particles simulated already, simulate just one more history (block) and report (kernel call would fail if total_histories &lt; or == 0).</span>
<a name="l00803"></a>00803       }     
<a name="l00804"></a>00804      
<a name="l00805"></a>00805     }   <span class="comment">// [Done with case of simulating projections by time or first projection by number of particles]    </span>
<a name="l00806"></a>00806     
<a name="l00807"></a>00807     <span class="comment">// else  ==&gt;  if using only 1 GPU and a fixed number of histories the whole speed test is skipped. The random seed will be different for each projection because it is updated after calling the kernel below.</span>
<a name="l00808"></a>00808   
<a name="l00809"></a>00809   
<a name="l00810"></a>00810     <span class="comment">// fflush(stdout); </span>
<a name="l00811"></a>00811     <span class="comment">// MPI_Barrier(MPI_COMM_WORLD);   // Synchronize MPI threads here if we want to have a better organized output text at the expense of losing some performance </span>
<a name="l00812"></a>00812 
<a name="l00813"></a>00813 
<a name="l00814"></a>00814   
<a name="l00815"></a>00815     <span class="comment">// *** Perform the MC simulation itself (the speed test would be skipped for a single CPU thread using a fix number of histories):</span>
<a name="l00816"></a>00816   
<a name="l00817"></a>00817     <span class="comment">// -- Compute the number of CUDA blocks to simulate, rounding up and making sure it is below the limit of 65535 blocks.</span>
<a name="l00818"></a>00818     <span class="comment">//    The total number of particles simulated will be increased to the nearest multiple &quot;histories_per_thread&quot;.</span>
<a name="l00819"></a>00819     <span class="keywordtype">int</span> total_threads = (int)(((<span class="keywordtype">double</span>)total_histories)/((<span class="keywordtype">double</span>)histories_per_thread) + 0.9990);     <span class="comment">// Divide the histories among GPU threads, rounding up</span>
<a name="l00820"></a>00820     <span class="keywordtype">int</span> total_threads_blocks = (int)(((<span class="keywordtype">double</span>)total_threads)/((<span class="keywordtype">double</span>)num_threads_per_block) + 0.9990);   <span class="comment">// Divide the GPU threads among CUDA blocks, rounding up</span>
<a name="l00821"></a>00821     <span class="keywordflow">if</span> (total_threads_blocks&gt;65535)
<a name="l00822"></a>00822     {     
<a name="l00823"></a>00823 <span class="preprocessor">      #ifdef USING_MPI       </span>
<a name="l00824"></a>00824 <span class="preprocessor"></span>        printf(<span class="stringliteral">&quot;          WARNING (MPI process #%d): %d hist per thread would produce %d CUDA blocks (&gt;65535 maximum).&quot;</span>, myID, histories_per_thread, total_threads_blocks);
<a name="l00825"></a>00825 <span class="preprocessor">      #else</span>
<a name="l00826"></a>00826 <span class="preprocessor"></span>        printf(<span class="stringliteral">&quot;\n          WARNING: %d hist per thread would produce %d CUDA blocks, more than the maximum value of 65535.&quot;</span>, histories_per_thread, total_threads_blocks);
<a name="l00827"></a>00827 <span class="preprocessor">      #endif</span>
<a name="l00828"></a>00828 <span class="preprocessor"></span>      total_threads_blocks = 65000;    <span class="comment">// Increase the histories per thread to have exactly 65000 blocks.</span>
<a name="l00829"></a>00829       histories_per_thread = (int) ( ((<span class="keywordtype">double</span>)total_histories)/((double)(total_threads_blocks*num_threads_per_block)) + 0.9990 );        
<a name="l00830"></a>00830       printf(<span class="stringliteral">&quot; Increasing to %d hist to run exactly %d blocks in the GPU.\n&quot;</span>, histories_per_thread, total_threads_blocks);
<a name="l00831"></a>00831     }
<a name="l00832"></a>00832     <span class="keywordflow">else</span> <span class="keywordflow">if</span> (total_threads_blocks&lt;1)
<a name="l00833"></a>00833     {
<a name="l00834"></a>00834       total_threads_blocks = 1;        <span class="comment">// Make sure we have at least 1 block to run</span>
<a name="l00835"></a>00835     }      
<a name="l00836"></a>00836     
<a name="l00837"></a>00837     total_histories = ((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)(total_threads_blocks*num_threads_per_block))*histories_per_thread;   <span class="comment">// Total histories will be equal or higher than the input value due to the rounding up in the division of the histories</span>
<a name="l00838"></a>00838     <span class="keywordtype">float</span> total_histories_current_kernel_float = (float)total_histories;   <span class="comment">// Keep a float approx of the num histories for the timing below</span>
<a name="l00839"></a>00839     
<a name="l00840"></a>00840 <span class="preprocessor">    #ifdef USING_MPI  </span>
<a name="l00841"></a>00841 <span class="preprocessor"></span>      <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n&quot;</span>);
<a name="l00842"></a>00842       printf(<span class="stringliteral">&quot;        ==&gt; CUDA (MPI process #%d in \&quot;%s\&quot;): Executing %d blocks of %d threads, with %d histories in each thread: %lld histories in total (random seed: %d).\n&quot;</span>, myID, MPI_processor_name, total_threads_blocks, num_threads_per_block, histories_per_thread, total_histories, seed_input);
<a name="l00843"></a>00843 <span class="preprocessor">    #else</span>
<a name="l00844"></a>00844 <span class="preprocessor"></span>      printf(<span class="stringliteral">&quot;\n        ==&gt; CUDA: Executing %d blocks of %d threads, with %d histories in each thread: %lld histories in total (random seed: %d).\n&quot;</span>, total_threads_blocks, num_threads_per_block, histories_per_thread, total_histories, seed_input);     
<a name="l00845"></a>00845 <span class="preprocessor">    #endif</span>
<a name="l00846"></a>00846 <span class="preprocessor"></span>    fflush(stdout); 
<a name="l00847"></a>00847     
<a name="l00848"></a>00848     <span class="comment">// -- Setup the execution parameters (Max number threads per block: 512, Max sizes each dimension of grid: 65535x65535x1)</span>
<a name="l00849"></a>00849 
<a name="l00850"></a>00850     dim3 blocks(total_threads_blocks, 1);
<a name="l00851"></a>00851     dim3 threads(num_threads_per_block, 1); 
<a name="l00852"></a>00852     
<a name="l00853"></a>00853     clock_kernel = clock();
<a name="l00854"></a>00854 
<a name="l00855"></a>00855     
<a name="l00856"></a>00856     <span class="comment">// *** Execute the x-ray transport kernel in the GPU ***</span>
<a name="l00857"></a>00857     track_particles&lt;&lt;&lt;blocks,threads&gt;&gt;&gt;(histories_per_thread, num_p, seed_input, image_device, voxels_Edep_device, voxel_mat_dens_device, mfp_Woodcock_table_device, mfp_table_a_device, mfp_table_b_device, rayleigh_table_device, compton_table_device, detector_data_device, source_data_device, materials_dose_device);
<a name="l00858"></a>00858     
<a name="l00859"></a>00859     
<a name="l00860"></a>00860     <span class="keywordflow">if</span> (1==doing_speed_test)
<a name="l00861"></a>00861       total_histories += histories_speed_test;     <span class="comment">// Speed test was done: compute the total number of histories including the particles simulated in the speed test </span>
<a name="l00862"></a>00862       
<a name="l00863"></a>00863     <span class="comment">// -- Move the pseudo-random number generator seed ahead to skip all the random numbers generated in the current projection by this and the other</span>
<a name="l00864"></a>00864     <span class="comment">//    &quot;numprocs&quot; MPI threads. Each projection will use independent seeds! (this code runs in parallel with the asynchronous GPU kernel):</span>
<a name="l00865"></a>00865     <a class="code" href="MC-GPU__v1_83_8cu.html#a6577c51b64b94b13c4e95df776275b5a" title="Initialize the first seed of the pseudo-random number generator (PRNG) RANECU to a position far away ...">update_seed_PRNG</a>(numprocs, total_histories, &amp;seed_input);   <span class="comment">// Do not repeat seed for each projection. Note that this function only updates 1 seed, the other is not computed.</span>
<a name="l00866"></a>00866               
<a name="l00867"></a>00867 
<a name="l00868"></a>00868 <span class="preprocessor">    #ifdef USING_MPI </span>
<a name="l00869"></a>00869 <span class="preprocessor"></span>      <span class="keywordflow">if</span> (numprocs&gt;1)  <span class="comment">// Using more than 1 MPI thread:</span>
<a name="l00870"></a>00870       {
<a name="l00871"></a>00871         <span class="comment">// -- Compute the total number of histories simulated with all MPI thread, including the speed test (histories_speed_test==0 if speed test was skipped).</span>
<a name="l00872"></a>00872         <span class="comment">//    These MPI messajes are sent concurrently with the GPU kernel computation for maximum efficiency.</span>
<a name="l00873"></a>00873         <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> current_GPU_histories = total_histories;  
<a name="l00874"></a>00874         return_reduce = MPI_Reduce(&amp;current_GPU_histories, &amp;total_histories, 1, MPI_UNSIGNED_LONG, MPI_SUM, 0, MPI_COMM_WORLD);  <span class="comment">// Sum all the simulated particles and send to thread 0</span>
<a name="l00875"></a>00875                 
<a name="l00876"></a>00876         <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l00877"></a>00877         {
<a name="l00878"></a>00878           <span class="keywordflow">if</span> (MPI_SUCCESS != return_reduce)
<a name="l00879"></a>00879             printf(<span class="stringliteral">&quot;\n\n !!ERROR!! Error getting the total number of particles simulated in all the GPUs (MPI_Reduce). return_reduce = %d.\n\n\n&quot;</span>, return_reduce);
<a name="l00880"></a>00880           
<a name="l00881"></a>00881           <span class="keywordflow">if</span> (1==simulating_by_time || 1==doing_speed_test)
<a name="l00882"></a>00882           {
<a name="l00883"></a>00883             printf(<span class="stringliteral">&quot;\n       -- Total number of histories being simulated in all the GPUs for the current projection (including speed test)= %.3lld.\n\n&quot;</span>, total_histories);
<a name="l00884"></a>00884             fflush(stdout);
<a name="l00885"></a>00885           }
<a name="l00886"></a>00886         }
<a name="l00887"></a>00887       }
<a name="l00888"></a>00888 <span class="preprocessor">    #endif</span>
<a name="l00889"></a>00889 <span class="preprocessor"></span>    
<a name="l00890"></a>00890     cudaThreadSynchronize();    <span class="comment">// Force the runtime to wait until the GPU kernel is completed</span>
<a name="l00891"></a>00891     getLastCudaError(<span class="stringliteral">&quot;\n\n !!Kernel execution failed while simulating particle tracks!! &quot;</span>);  <span class="comment">// Check if kernel execution generated any error</span>
<a name="l00892"></a>00892 
<a name="l00893"></a>00893     <span class="keywordtype">float</span> real_GPU_speed = total_histories_current_kernel_float/(float(clock()-clock_kernel)/CLOCKS_PER_SEC);  <span class="comment">// GPU speed for all the image simulation, not just the speed test.</span>
<a name="l00894"></a>00894     
<a name="l00895"></a>00895 <span class="comment">//     #ifdef USING_MPI   </span>
<a name="l00896"></a>00896 <span class="comment">//       printf(&quot;        ==&gt; CUDA (MPI process #%d in \&quot;%s\&quot;): GPU kernel execution time: %.4f s (%.3f hist/s)\n&quot;, myID, MPI_processor_name, time_kernel, total_histories_current_kernel_float/time_kernel);</span>
<a name="l00897"></a>00897 <span class="comment">//     #else     </span>
<a name="l00898"></a>00898 <span class="comment">//       printf(&quot;        ==&gt; CUDA: Kernel execution time: %.4f s\n&quot;, time_kernel);</span>
<a name="l00899"></a>00899 <span class="comment">//     #endif  </span>
<a name="l00900"></a>00900       
<a name="l00901"></a>00901       
<a name="l00902"></a>00902     <span class="comment">// -- Copy the simulated image from the GPU memory to the CPU:           </span>
<a name="l00903"></a>00903     checkCudaErrors(cudaMemcpy(image, image_device, image_bytes, cudaMemcpyDeviceToHost) );  <span class="comment">// Copy final results to host</span>
<a name="l00904"></a>00904 
<a name="l00905"></a>00905          <span class="comment"></span>
<a name="l00906"></a>00906 <span class="comment">///////////////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00907"></a>00907 <span class="comment"></span>  
<a name="l00908"></a>00908     
<a name="l00909"></a>00909 <span class="preprocessor">#else</span>
<a name="l00910"></a>00910 <span class="preprocessor"></span>    
<a name="l00911"></a>00911     <span class="comment">// *** Executing the kernel in the CPU:</span>
<a name="l00912"></a>00912     
<a name="l00913"></a>00913     <span class="comment">//     If using more than one MPI thread, the number of particles is equally dividied among the threads.  </span>
<a name="l00914"></a>00914     <span class="comment">//    !!DeBuG!! --&gt; NOT USING SPEED TEST IN THE CPU!! Not possible to limit the execution by time in the CPU.</span>
<a name="l00915"></a>00915     
<a name="l00916"></a>00916     <span class="keywordtype">int</span> total_threads = (int)(((<span class="keywordtype">double</span>)total_histories)/((double)histories_per_thread*numprocs) + 0.9990);     <span class="comment">// Divide the histories among MPI threads, rounding up</span>
<a name="l00917"></a>00917     <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories_per_thread = ((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int)(total_threads))*histories_per_thread;
<a name="l00918"></a>00918     total_histories = total_histories_per_thread*numprocs;    <span class="comment">// Total histories will be equal or higher than the input value due to the rounding up in the division of the histories       </span>
<a name="l00919"></a>00919     
<a name="l00920"></a>00920     
<a name="l00921"></a>00921     <span class="keywordflow">if</span> (numprocs&gt;1) 
<a name="l00922"></a>00922     {
<a name="l00923"></a>00923 <span class="preprocessor">      #ifdef USING_MPI       </span>
<a name="l00924"></a>00924 <span class="preprocessor"></span>        <a class="code" href="MC-GPU__v1_83_8cu.html#a6577c51b64b94b13c4e95df776275b5a" title="Initialize the first seed of the pseudo-random number generator (PRNG) RANECU to a position far away ...">update_seed_PRNG</a>(myID, total_histories, &amp;seed_input);   <span class="comment">// Compute the initial random seed for each MPI threads, avoiding overlapping of the random sequences</span>
<a name="l00925"></a>00925       
<a name="l00926"></a>00926         printf(<span class="stringliteral">&quot;       Executing %d history batches in the CPU, with %d histories in each batch  (thread %d of %d at \&#39;%s\&#39;): %lld histories (random seed=%d).\n&quot;</span>, total_threads, histories_per_thread, myID+1, numprocs, MPI_processor_name, total_histories_per_thread, seed_input);
<a name="l00927"></a>00927         <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Simulating %lld histories in total for the %d MPI threads.\n\n&quot;</span>, total_histories, numprocs);
<a name="l00928"></a>00928 <span class="preprocessor">      #endif</span>
<a name="l00929"></a>00929 <span class="preprocessor"></span>    }
<a name="l00930"></a>00930     <span class="keywordflow">else</span>
<a name="l00931"></a>00931     {
<a name="l00932"></a>00932       printf(<span class="stringliteral">&quot;       Executing %d history batches in the CPU, with %d histories in each batch: %lld histories in total.\n\n&quot;</span>, total_threads, histories_per_thread, total_histories);
<a name="l00933"></a>00933     }    
<a name="l00934"></a>00934     fflush(stdout); 
<a name="l00935"></a>00935     
<a name="l00936"></a>00936 
<a name="l00937"></a>00937     <span class="comment">// -- Copy local structures to global struct variables accessible from &quot;track_particles&quot; (__constant__ variables in the GPU):</span>
<a name="l00938"></a>00938     <a class="code" href="MC-GPU__v1_83_8h.html#ad18b13e5db1d8c7c4acdd9af492d6914" title="Global variable to be stored in the GPU constant memory defining the source energy spectrum...">source_energy_data_CONST</a> = source_energy_data;
<a name="l00939"></a>00939     <a class="code" href="MC-GPU__v1_83_8h.html#a129396d7723d11e10c673c1171773cdc" title="Global variable to be stored in the GPU constant memory defining the size of the voxel phantom...">voxel_data_CONST</a> = voxel_data;
<a name="l00940"></a>00940     <a class="code" href="MC-GPU__v1_83_8h.html#a8a3e410fb23d04f8a27d3d71c2381d24" title="Global variable to be stored in the GPU constant memory defining the linear interpolation data...">mfp_table_data_CONST</a> = mfp_table_data;
<a name="l00941"></a>00941     <a class="code" href="MC-GPU__v1_83_8h.html#a5d34c13b457e76dec83755628ead56fc" title="Global variable to be stored in the GPU constant memory defining the coordinates of the dose depositi...">dose_ROI_x_min_CONST</a> = dose_ROI_x_min;
<a name="l00942"></a>00942     <a class="code" href="MC-GPU__v1_83_8h.html#a3c22177ce0c73c17ca8716d997678b61">dose_ROI_x_max_CONST</a> = dose_ROI_x_max;
<a name="l00943"></a>00943     <a class="code" href="MC-GPU__v1_83_8h.html#ac161c8f74fd460e82e73917b3afff5fc">dose_ROI_y_min_CONST</a> = dose_ROI_y_min;
<a name="l00944"></a>00944     <a class="code" href="MC-GPU__v1_83_8h.html#ab591ece7d8e024dcc8de71f59bc1cea5">dose_ROI_y_max_CONST</a> = dose_ROI_y_max;
<a name="l00945"></a>00945     <a class="code" href="MC-GPU__v1_83_8h.html#a81db18966d58018976252f898cd905d9">dose_ROI_z_min_CONST</a> = dose_ROI_z_min;
<a name="l00946"></a>00946     <a class="code" href="MC-GPU__v1_83_8h.html#aea8fcb598ccc9c18ae601f9c7d936b9d">dose_ROI_z_max_CONST</a> = dose_ROI_z_max;
<a name="l00947"></a>00947 
<a name="l00948"></a>00948     
<a name="l00949"></a>00949     <span class="keywordtype">int</span> CPU_batch;
<a name="l00950"></a>00950     <span class="keywordflow">for</span>(CPU_batch=0; CPU_batch&lt;total_threads; CPU_batch++)
<a name="l00951"></a>00951     {
<a name="l00952"></a>00952       <span class="comment">// -- Simulate a particle track initializing the PRNG with the particle number &#39;n&#39;:</span>
<a name="l00953"></a>00953       <a class="code" href="MC-GPU__kernel__v1_83_8cu.html#ab1e2e311c809fe74d093498e53c9c954" title="Initialize the image array, ie, set all pixels to zero Essentially, this function has the same effect...">track_particles</a>(CPU_batch, histories_per_thread, num_p, seed_input, image, voxels_Edep, voxel_mat_dens, mfp_Woodcock_table, mfp_table_a, mfp_table_b, &amp;rayleigh_table, &amp;compton_table, detector_data, source_data, materials_dose);
<a name="l00954"></a>00954     }
<a name="l00955"></a>00955 
<a name="l00956"></a>00956     
<a name="l00957"></a>00957 <span class="preprocessor">#endif    </span>
<a name="l00958"></a>00958 <span class="preprocessor"></span>
<a name="l00959"></a>00959 
<a name="l00960"></a>00960     <span class="comment">// Get current time and calculate execution time in the MC loop:</span>
<a name="l00961"></a>00961     time_elapsed_MC_loop = ((double)(clock()-clock_start))/CLOCKS_PER_SEC;       
<a name="l00962"></a>00962     time_total_MC_simulation += time_elapsed_MC_loop;   <span class="comment">// Count total time (in seconds).</span>
<a name="l00963"></a>00963         <span class="comment">//  printf(&quot;\n    -- MONTE CARLO LOOP finished: time tallied in MAIN program: %.3f s\n\n&quot;, time_elapsed_MC_loop);</span>
<a name="l00964"></a>00964         
<a name="l00965"></a>00965 
<a name="l00966"></a>00966 <span class="comment"></span>
<a name="l00967"></a>00967 <span class="comment">///////////////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l00968"></a>00968 <span class="comment"></span>     
<a name="l00969"></a>00969 
<a name="l00970"></a>00970     <span class="comment">// *** Move the images simulated in the GPU (or multiple CPU cores) to the host memory space:</span>
<a name="l00971"></a>00971     
<a name="l00972"></a>00972 <span class="preprocessor">#ifdef USING_MPI </span>
<a name="l00973"></a>00973 <span class="preprocessor"></span>    <span class="keywordflow">if</span> (numprocs&gt;1)  <span class="comment">// Using more than 1 MPI thread</span>
<a name="l00974"></a>00974     {
<a name="l00975"></a>00975       <span class="comment">// -- Add the images simulated in all the MPI threads:      </span>
<a name="l00976"></a>00976       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n        &gt;&gt;  Synchronize the MPI threads and accumulate the simulated images (MPI_Reduce).\n\n&quot;</span>);                    
<a name="l00977"></a>00977       
<a name="l00978"></a>00978       <span class="comment">// Allocate the memory for the final image in the master thread:</span>
<a name="l00979"></a>00979       <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> *image_MPI = NULL;
<a name="l00980"></a>00980       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> image_MPI = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>*) malloc(image_bytes);
<a name="l00981"></a>00981       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> <span class="keywordflow">if</span> (image_MPI==NULL)
<a name="l00982"></a>00982       {
<a name="l00983"></a>00983         printf(<span class="stringliteral">&quot;\n\n   !!malloc ERROR!! Problem allocating the total MPI image. Out of memory??\n\n&quot;</span>);  
<a name="l00984"></a>00984         exit(-4);
<a name="l00985"></a>00985       }
<a name="l00986"></a>00986 
<a name="l00987"></a>00987 
<a name="l00988"></a>00988       <span class="comment">// !!DeBuG!! To know how much time the threads lose waiting for other threads in the MPI_Reduce, I have to use an explicit barrier here. It may be more efficient to let the threads advance to the MPI_Reduce directly.</span>
<a name="l00989"></a>00989       clock_start = clock();      
<a name="l00990"></a>00990       MPI_Barrier(MPI_COMM_WORLD);   <span class="comment">// Synchronize MPI threads            </span>
<a name="l00991"></a>00991       
<a name="l00992"></a>00992       current_time=time(NULL);      
<a name="l00993"></a>00993       char_time = ctime(&amp;current_time); char_time[19] = <span class="charliteral">&#39;\0&#39;</span>;   <span class="comment">// The time is located between the characters 11 and 19.</span>
<a name="l00994"></a>00994       
<a name="l00995"></a>00995       
<a name="l00996"></a>00996 <span class="preprocessor">    #ifdef USING_CUDA</span>
<a name="l00997"></a>00997 <span class="preprocessor"></span>      <span class="keywordflow">if</span> (1==doing_speed_test)    <span class="comment">// This message will be shown only for the first projection simulated in the GPU.</span>
<a name="l00998"></a>00998         printf(<span class="stringliteral">&quot;        ==&gt; CUDA (MPI process #%d in \&quot;%s\&quot;): GPU speed = %.4f hist/s. Time spent at MPI_Barrier waiting to add the partial images: %.6f s (time: %8s)\n&quot;</span>, myID, MPI_processor_name, real_GPU_speed, ((<span class="keywordtype">double</span>)(clock()-clock_start))/CLOCKS_PER_SEC, &amp;char_time[11]);
<a name="l00999"></a>00999 <span class="preprocessor">    #else        </span>
<a name="l01000"></a>01000 <span class="preprocessor"></span>      <span class="keywordflow">if</span> (-1==doing_speed_test)
<a name="l01001"></a>01001       {
<a name="l01002"></a>01002         printf(<span class="stringliteral">&quot;        ==&gt; CUDA (MPI process #%d in \&quot;%s\&quot;): Time spent at MPI_Barrier waiting to add the partial images: %.6f s (time: %8s)\n&quot;</span>, myID, MPI_processor_name, ((<span class="keywordtype">double</span>)(clock()-clock_start))/CLOCKS_PER_SEC, &amp;char_time[11]);
<a name="l01003"></a>01003         doing_speed_test = 0;
<a name="l01004"></a>01004       }
<a name="l01005"></a>01005 <span class="preprocessor">    #endif  </span>
<a name="l01006"></a>01006 <span class="preprocessor"></span>      
<a name="l01007"></a>01007       
<a name="l01008"></a>01008       fflush(stdout);      
<a name="l01009"></a>01009       
<a name="l01010"></a>01010       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> clock_start = clock();
<a name="l01011"></a>01011                       
<a name="l01012"></a>01012       <span class="comment">// -- Sum the pixel values from the different simulated images and send to thread 0.</span>
<a name="l01013"></a>01013       <span class="comment">//    MPI_Reduce will act as a synchronization barrier for all the MPI threads.</span>
<a name="l01014"></a>01014       <span class="keywordtype">int</span> num_pixels_image = image_bytes/((int)<span class="keyword">sizeof</span>(<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>));   <span class="comment">// Number of elements allocated in the &quot;image&quot; array.         </span>
<a name="l01015"></a>01015       return_reduce = MPI_Reduce(image, image_MPI, num_pixels_image, MPI_UNSIGNED_LONG, MPI_SUM, 0, MPI_COMM_WORLD); 
<a name="l01016"></a>01016       
<a name="l01017"></a>01017       <span class="keywordflow">if</span> (MPI_SUCCESS != return_reduce)
<a name="l01018"></a>01018       {
<a name="l01019"></a>01019         printf(<span class="stringliteral">&quot;\n\n !!ERROR!! Possible error reducing (MPI_SUM) the image results??? Returned value MPI_Reduce = %d\n\n\n&quot;</span>, return_reduce);
<a name="l01020"></a>01020       }
<a name="l01021"></a>01021               
<a name="l01022"></a>01022       <span class="comment">// -- Exchange the image simulated in thread 0 for the final image from all threads, in the master thread:</span>
<a name="l01023"></a>01023       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l01024"></a>01024       {
<a name="l01025"></a>01025         free(image);
<a name="l01026"></a>01026         image = image_MPI;    <span class="comment">// point the image pointer to the new image in host memory</span>
<a name="l01027"></a>01027         image_MPI = NULL;                
<a name="l01028"></a>01028 
<a name="l01029"></a>01029         printf(<span class="stringliteral">&quot;\n       -- Time reducing the images simulated by all the MPI threads (MPI_Reduce) according to the master thread = %.6f s.\n&quot;</span>, ((<span class="keywordtype">double</span>)(clock()-clock_start))/CLOCKS_PER_SEC); 
<a name="l01030"></a>01030       }
<a name="l01031"></a>01031     }
<a name="l01032"></a>01032 <span class="preprocessor">#endif</span>
<a name="l01033"></a>01033 <span class="preprocessor"></span>                
<a name="l01034"></a>01034 
<a name="l01035"></a>01035     <span class="comment">// *** Report the final results:</span>
<a name="l01036"></a>01036     <span class="keywordtype">char</span> file_name_output_num_p[253];
<a name="l01037"></a>01037     <span class="keywordflow">if</span> (1==num_projections)
<a name="l01038"></a>01038       strcpy(file_name_output_num_p, file_name_output);   <span class="comment">// Use the input name for single projection</span>
<a name="l01039"></a>01039     <span class="keywordflow">else</span>
<a name="l01040"></a>01040       sprintf(file_name_output_num_p, <span class="stringliteral">&quot;%s_%04d&quot;</span>, file_name_output, num_p);   <span class="comment">// Create the output file name with the input name + projection number (4 digits, padding with 0)</span>
<a name="l01041"></a>01041 
<a name="l01042"></a>01042     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> <a class="code" href="MC-GPU__v1_83_8cu.html#aa23f71412757b1d0e1d4c00e6b8a0c18" title="Report the tallied image in ASCII and binary form (32-bit floats).">report_image</a>(file_name_output_num_p, detector_data, source_data, mean_energy_spectrum, image, time_elapsed_MC_loop, total_histories, num_p, num_projections, D_angle, initial_angle, myID, numprocs);
<a name="l01043"></a>01043 
<a name="l01044"></a>01044     <span class="comment">// *** Clear the image after reporting, unless this is the last projection to simulate:</span>
<a name="l01045"></a>01045     <span class="keywordflow">if</span> (num_p&lt;(num_projections-1))
<a name="l01046"></a>01046     {
<a name="l01047"></a>01047       <span class="keywordtype">int</span> pixels_per_image = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a> * detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>;
<a name="l01048"></a>01048 <span class="preprocessor">      #ifdef USING_CUDA</span>
<a name="l01049"></a>01049 <span class="preprocessor"></span>        <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Launching kernel to reset the device image to 0: number of blocks = %d, threads per block = 128\n&quot;</span>, (<span class="keywordtype">int</span>)(ceil(pixels_per_image/128.0f)+0.01f) );
<a name="l01050"></a>01050         init_image_array_GPU&lt;&lt;&lt;(int)(ceil(pixels_per_image/128.0f)+0.01f),128&gt;&gt;&gt;(image_device, pixels_per_image);
<a name="l01051"></a>01051         cudaThreadSynchronize();
<a name="l01052"></a>01052         getLastCudaError(<span class="stringliteral">&quot;\n\n !!Kernel execution failed initializing the image array!! &quot;</span>);  <span class="comment">// Check if kernel execution generated any error:</span>
<a name="l01053"></a>01053 <span class="preprocessor">      #else        </span>
<a name="l01054"></a>01054 <span class="preprocessor"></span>        memset(image, 0, image_bytes);     <span class="comment">//   Init memory space to 0.  (see http://www.lainoox.com/c-memset-examples/)</span>
<a name="l01055"></a>01055 <span class="preprocessor">      #endif</span>
<a name="l01056"></a>01056 <span class="preprocessor"></span>    }
<a name="l01057"></a>01057     
<a name="l01058"></a>01058   }  <span class="comment">// [Projection loop end: iterate for next CT projection angle]</span>
<a name="l01059"></a>01059 
<a name="l01060"></a>01060 <span class="comment"></span>
<a name="l01061"></a>01061 <span class="comment">///////////////////////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01062"></a>01062 <span class="comment"></span>          
<a name="l01063"></a>01063      
<a name="l01064"></a>01064   <span class="comment">// *** Simulation finished! Report dose and timings and clean up.</span>
<a name="l01065"></a>01065 
<a name="l01066"></a>01066 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l01067"></a>01067 <span class="preprocessor"></span>  <span class="keywordflow">if</span> (dose_ROI_x_max &gt; -1)
<a name="l01068"></a>01068   {   
<a name="l01069"></a>01069     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> clock_kernel = clock();    
<a name="l01070"></a>01070 
<a name="l01071"></a>01071     checkCudaErrors( cudaMemcpy( voxels_Edep, voxels_Edep_device, voxels_Edep_bytes, cudaMemcpyDeviceToHost) );  <span class="comment">// Copy final dose results to host (for every MPI threads)</span>
<a name="l01072"></a>01072 
<a name="l01073"></a>01073     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Time copying dose results from device to host: %.6f s\n&quot;</span>, <span class="keywordtype">float</span>(clock()-clock_kernel)/CLOCKS_PER_SEC);
<a name="l01074"></a>01074   }
<a name="l01075"></a>01075   
<a name="l01076"></a>01076   <span class="keywordflow">if</span> (flag_material_dose==1)
<a name="l01077"></a>01077     checkCudaErrors( cudaMemcpy( materials_dose, materials_dose_device, MAX_MATERIALS*<span class="keyword">sizeof</span>(<a class="code" href="structulonglong2.html">ulonglong2</a>), cudaMemcpyDeviceToHost) );  <span class="comment">// Copy materials dose results to host, if tally enabled in input file.   !!tally_materials_dose!!</span>
<a name="l01078"></a>01078 
<a name="l01079"></a>01079   <span class="comment">// -- Clean up GPU device memory:</span>
<a name="l01080"></a>01080   clock_kernel = clock();    
<a name="l01081"></a>01081 
<a name="l01082"></a>01082   cudaFree(voxel_mat_dens_device);
<a name="l01083"></a>01083   cudaFree(image_device);
<a name="l01084"></a>01084   cudaFree(mfp_Woodcock_table_device);
<a name="l01085"></a>01085   cudaFree(mfp_table_a_device);
<a name="l01086"></a>01086   cudaFree(mfp_table_b_device);
<a name="l01087"></a>01087   cudaFree(voxels_Edep_device);
<a name="l01088"></a>01088   checkCudaErrors( cudaThreadExit() );
<a name="l01089"></a>01089 
<a name="l01090"></a>01090   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Time freeing the device memory and ending the GPU threads: %.6f s\n&quot;</span>, <span class="keywordtype">float</span>(clock()-clock_kernel)/CLOCKS_PER_SEC);
<a name="l01091"></a>01091 
<a name="l01092"></a>01092 <span class="preprocessor">#endif</span>
<a name="l01093"></a>01093 <span class="preprocessor"></span>
<a name="l01094"></a>01094 
<a name="l01095"></a>01095 <span class="preprocessor">#ifdef USING_MPI</span>
<a name="l01096"></a>01096 <span class="preprocessor"></span>  current_time=time(NULL);     <span class="comment">// Get current time (in seconds)</span>
<a name="l01097"></a>01097   char_time = ctime(&amp;current_time); char_time[19] = <span class="charliteral">&#39;\0&#39;</span>;   <span class="comment">// The time is located betwen the characters 11 and 19.  </span>
<a name="l01098"></a>01098   printf(<span class="stringliteral">&quot;        &gt;&gt; MPI thread %d in \&quot;%s\&quot; done! (local time: %s)\n&quot;</span>, myID, MPI_processor_name, &amp;char_time[11]);
<a name="l01099"></a>01099   fflush(stdout);   <span class="comment">// Clear the screen output buffer</span>
<a name="l01100"></a>01100 <span class="preprocessor">#endif</span>
<a name="l01101"></a>01101 <span class="preprocessor"></span>
<a name="l01102"></a>01102 
<a name="l01103"></a>01103   
<a name="l01104"></a>01104   <span class="comment">// *** Report the total dose for all the projections, if the tally is not disabled (must be done after MPI_Barrier to have all the MPI threads synchronized):</span>
<a name="l01105"></a>01105   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> clock_start = clock(); 
<a name="l01106"></a>01106   
<a name="l01107"></a>01107   <span class="keywordflow">if</span> (dose_ROI_x_max &gt; -1)
<a name="l01108"></a>01108   {   
<a name="l01109"></a>01109     
<a name="l01110"></a>01110 <span class="preprocessor">#ifdef USING_MPI</span>
<a name="l01111"></a>01111 <span class="preprocessor"></span>    <span class="keywordflow">if</span> (numprocs&gt;1)
<a name="l01112"></a>01112     {
<a name="l01113"></a>01113       <span class="comment">// -- Use MPI_Reduce to accumulate the dose from all projections:      </span>
<a name="l01114"></a>01114       <span class="comment">//    Allocate memory in the root node to combine the dose results with MPI_REDUCE:</span>
<a name="l01115"></a>01115       <span class="keywordtype">int</span> num_voxels_ROI = voxels_Edep_bytes/((int)<span class="keyword">sizeof</span>(<a class="code" href="structulonglong2.html">ulonglong2</a>));   <span class="comment">// Number of elements allocated in the &quot;dose&quot; array.</span>
<a name="l01116"></a>01116       <a class="code" href="structulonglong2.html">ulonglong2</a> *voxels_Edep_total = (<a class="code" href="structulonglong2.html">ulonglong2</a>*) malloc(voxels_Edep_bytes);
<a name="l01117"></a>01117       <span class="keywordflow">if</span> (voxels_Edep_total==NULL)
<a name="l01118"></a>01118       {
<a name="l01119"></a>01119         printf(<span class="stringliteral">&quot;\n\n   !!malloc ERROR!! Not enough memory to allocate %d voxels by the MPI root node for the total deposited dose (and uncertainty) array (%f Mbytes)!!\n\n&quot;</span>, num_voxels_ROI, voxels_Edep_bytes/(1024.f*1024.f));
<a name="l01120"></a>01120         exit(-2);
<a name="l01121"></a>01121       }
<a name="l01122"></a>01122       <span class="keywordflow">else</span>
<a name="l01123"></a>01123       {
<a name="l01124"></a>01124         <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a>
<a name="l01125"></a>01125         {
<a name="l01126"></a>01126           printf(<span class="stringliteral">&quot;\n        &gt;&gt; Array for the total deposited dose correctly allocated by the MPI root node (%f Mbytes).\n&quot;</span>, voxels_Edep_bytes/(1024.f*1024.f));
<a name="l01127"></a>01127           printf(  <span class="stringliteral">&quot;           Waiting at MPI_Barrier for thread synchronization.\n&quot;</span>);
<a name="l01128"></a>01128         }
<a name="l01129"></a>01129       }      
<a name="l01130"></a>01130       
<a name="l01131"></a>01131       
<a name="l01132"></a>01132       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n        &gt;&gt; Calling MPI_Reduce to accumulate the dose from all projections...\n\n&quot;</span>);    
<a name="l01133"></a>01133       
<a name="l01134"></a>01134       return_reduce = MPI_Reduce(voxels_Edep, voxels_Edep_total, 2*num_voxels_ROI, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);   <span class="comment">// Sum all the doses in &quot;voxels_Edep_total&quot; at thread 0.</span>
<a name="l01135"></a>01135             <span class="comment">// !!DeBuG!! I am sending a &quot;ulonglong2&quot; array as if it was composed of 2 &quot;ulonglong&quot; variables per element. There could be problems if the alignment in the structure includes some extra padding space (but it seems ok for a 64-bit computer).</span>
<a name="l01136"></a>01136       <span class="keywordflow">if</span> (MPI_SUCCESS != return_reduce)
<a name="l01137"></a>01137       {
<a name="l01138"></a>01138         printf(<span class="stringliteral">&quot;\n\n !!ERROR!! Possible error reducing (MPI_SUM) the dose results??? return_reduce = %d for thread %d\n\n\n&quot;</span>, return_reduce, myID);
<a name="l01139"></a>01139       }
<a name="l01140"></a>01140 
<a name="l01141"></a>01141       <span class="comment">// -- Exchange the dose simulated in thread 0 for the final dose from all threads  </span>
<a name="l01142"></a>01142       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a>
<a name="l01143"></a>01143       {
<a name="l01144"></a>01144         free(voxels_Edep);
<a name="l01145"></a>01145         voxels_Edep = voxels_Edep_total;    <span class="comment">// point the voxels_Edep pointer to the final voxels_Edep array in host memory</span>
<a name="l01146"></a>01146         voxels_Edep_total = NULL;           <span class="comment">// This pointer is not needed by now</span>
<a name="l01147"></a>01147       }
<a name="l01148"></a>01148     }
<a name="l01149"></a>01149 <span class="preprocessor">#endif</span>
<a name="l01150"></a>01150 <span class="preprocessor"></span>        
<a name="l01151"></a>01151     <span class="comment">// -- Report the total dose for all the projections:</span>
<a name="l01152"></a>01152     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> <a class="code" href="MC-GPU__v1_83_8cu.html#aeceeb305fc1d995c4730e0fcedf10d0c" title="Report the total tallied 3D voxel dose deposition for all projections.">report_voxels_dose</a>(file_dose_output, num_projections, &amp;voxel_data, voxel_mat_dens, voxels_Edep, time_total_MC_simulation, total_histories, dose_ROI_x_min, dose_ROI_x_max, dose_ROI_y_min, dose_ROI_y_max, dose_ROI_z_min, dose_ROI_z_max, source_data);        
<a name="l01153"></a>01153   }
<a name="l01154"></a>01154   
<a name="l01155"></a>01155   
<a name="l01156"></a>01156   <span class="comment">// -- Report &quot;tally_materials_dose&quot; with data from all MPI threads, if tally enabled:</span>
<a name="l01157"></a>01157   <span class="keywordflow">if</span> (flag_material_dose==1)
<a name="l01158"></a>01158   {
<a name="l01159"></a>01159 <span class="preprocessor">  #ifdef USING_MPI</span>
<a name="l01160"></a>01160 <span class="preprocessor"></span>    <a class="code" href="structulonglong2.html">ulonglong2</a> materials_dose_total[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];
<a name="l01161"></a>01161     return_reduce = MPI_Reduce(materials_dose, materials_dose_total, 2*MAX_MATERIALS, MPI_UNSIGNED_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);   <span class="comment">// !!tally_materials_dose!!</span>
<a name="l01162"></a>01162 <span class="preprocessor">  #else</span>
<a name="l01163"></a>01163 <span class="preprocessor"></span>    <a class="code" href="structulonglong2.html">ulonglong2</a> *materials_dose_total = materials_dose;  <span class="comment">// Create a dummy pointer to the materials_dose data </span>
<a name="l01164"></a>01164 <span class="preprocessor">  #endif</span>
<a name="l01165"></a>01165 <span class="preprocessor"></span>    
<a name="l01166"></a>01166     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> <a class="code" href="MC-GPU__v1_83_8cu.html#aa02ffc836f2b6f9de5386cadb11e30b7" title="Report the tallied dose to each material number, accounting for different densities in different regi...">report_materials_dose</a>(num_projections, total_histories, density_nominal, materials_dose_total, mass_materials);    <span class="comment">// Report the material dose  !!tally_materials_dose!!</span>
<a name="l01167"></a>01167   }
<a name="l01168"></a>01168   
<a name="l01169"></a>01169   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> clock_end = clock();
<a name="l01170"></a>01170   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n       ==&gt; CUDA: Time reporting the dose data: %.6f s\n&quot;</span>, ((<span class="keywordtype">double</span>)(clock_end-clock_start))/CLOCKS_PER_SEC);
<a name="l01171"></a>01171   
<a name="l01172"></a>01172 
<a name="l01173"></a>01173   <span class="comment">// *** Clean up RAM memory. If CUDA was used, the geometry and table data were already cleaned for MPI threads other than root after copying data to the GPU:</span>
<a name="l01174"></a>01174   free(voxels_Edep);
<a name="l01175"></a>01175   free(image);
<a name="l01176"></a>01176 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l01177"></a>01177 <span class="preprocessor"></span>  <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> free(voxel_mat_dens);
<a name="l01178"></a>01178 <span class="preprocessor">#else</span>
<a name="l01179"></a>01179 <span class="preprocessor"></span>  free(voxel_mat_dens);
<a name="l01180"></a>01180   free(mfp_Woodcock_table);
<a name="l01181"></a>01181   free(mfp_table_a);
<a name="l01182"></a>01182   free(mfp_table_b);
<a name="l01183"></a>01183 <span class="preprocessor">#endif</span>
<a name="l01184"></a>01184 <span class="preprocessor"></span>   
<a name="l01185"></a>01185 
<a name="l01186"></a>01186 <span class="preprocessor">#ifdef USING_MPI</span>
<a name="l01187"></a>01187 <span class="preprocessor"></span>  MPI_Finalize();   <span class="comment">// Finalize MPI library: no more MPI calls allowed below.</span>
<a name="l01188"></a>01188 <span class="preprocessor">#endif</span>
<a name="l01189"></a>01189 <span class="preprocessor"></span>  
<a name="l01190"></a>01190     
<a name="l01191"></a>01191   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l01192"></a>01192   {
<a name="l01193"></a>01193     printf(<span class="stringliteral">&quot;\n\n\n    -- SIMULATION FINISHED!\n&quot;</span>);
<a name="l01194"></a>01194     
<a name="l01195"></a>01195     time_total_MC_init_report = ((double)(clock()-clock_start_beginning))/CLOCKS_PER_SEC;
<a name="l01196"></a>01196 
<a name="l01197"></a>01197     <span class="comment">// -- Report total performance:</span>
<a name="l01198"></a>01198     printf(<span class="stringliteral">&quot;\n\n       ****** TOTAL SIMULATION PERFORMANCE (including initialization and reporting) ******\n\n&quot;</span>);  
<a name="l01199"></a>01199     printf(    <span class="stringliteral">&quot;          &gt;&gt;&gt; Execution time including initialization, transport and report: %.3f s.\n&quot;</span>, time_total_MC_init_report);
<a name="l01200"></a>01200     printf(    <span class="stringliteral">&quot;          &gt;&gt;&gt; Time spent in the Monte Carlo transport only: %.3f s.\n&quot;</span>, time_total_MC_simulation);
<a name="l01201"></a>01201     printf(    <span class="stringliteral">&quot;          &gt;&gt;&gt; Time spent in initialization, reporting and clean up: %.3f s.\n\n&quot;</span>, (time_total_MC_init_report-time_total_MC_simulation));
<a name="l01202"></a>01202     printf(    <span class="stringliteral">&quot;          &gt;&gt;&gt; Total number of simulated x rays:  %lld\n&quot;</span>, total_histories*((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)num_projections));      
<a name="l01203"></a>01203     <span class="keywordflow">if</span> (time_total_MC_init_report&gt;0.000001)
<a name="l01204"></a>01204       printf(  <span class="stringliteral">&quot;          &gt;&gt;&gt; Total speed (using %d thread, including initialization time) [x-rays/s]:  %.2f\n\n&quot;</span>, numprocs, (<span class="keywordtype">double</span>)(total_histories*((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)num_projections))/time_total_MC_init_report);
<a name="l01205"></a>01205   
<a name="l01206"></a>01206     current_time=time(NULL);     <span class="comment">// Get current time (in seconds)</span>
<a name="l01207"></a>01207     
<a name="l01208"></a>01208     printf(<span class="stringliteral">&quot;\n****** Code execution finished on: %s\n\n&quot;</span>, ctime(&amp;current_time));
<a name="l01209"></a>01209   }
<a name="l01210"></a>01210   
<a name="l01211"></a>01211 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l01212"></a>01212 <span class="preprocessor"></span>  cudaDeviceReset();  <span class="comment">// Destroy the CUDA context before ending program (flush visual debugger data).</span>
<a name="l01213"></a>01213 <span class="preprocessor">#endif</span>
<a name="l01214"></a>01214 <span class="preprocessor"></span>
<a name="l01215"></a>01215   <span class="keywordflow">return</span> 0;
<a name="l01216"></a>01216 }
<a name="l01217"></a>01217 
<a name="l01218"></a>01218 
<a name="l01219"></a>01219 
<a name="l01220"></a>01220 
<a name="l01221"></a>01221 <span class="comment"></span>
<a name="l01222"></a>01222 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01223"></a>01223 <span class="comment">//! Read the input file given in the command line and return the significant data.</span>
<a name="l01224"></a>01224 <span class="comment">//! Example input file:</span>
<a name="l01225"></a>01225 <span class="comment">//!</span>
<a name="l01226"></a>01226 <span class="comment">//!    1000000          [Total number of histories to simulate]</span>
<a name="l01227"></a>01227 <span class="comment">//!    geometry.vox     [Voxelized geometry file name]</span>
<a name="l01228"></a>01228 <span class="comment">//!    material.mat     [Material data file name]</span>
<a name="l01229"></a>01229 <span class="comment">//!</span>
<a name="l01230"></a>01230 <span class="comment">//!       @param[in] argc   Command line parameters</span>
<a name="l01231"></a>01231 <span class="comment">//!       @param[in] argv   Command line parameters: name of input file</span>
<a name="l01232"></a>01232 <span class="comment">//!       @param[out] total_histories  Total number of particles to simulate</span>
<a name="l01233"></a>01233 <span class="comment">//!       @param[out] seed_input   Input random number generator seed</span>
<a name="l01234"></a>01234 <span class="comment">//!       @param[out] num_threads_per_block   Number of CUDA threads for each GPU block</span>
<a name="l01235"></a>01235 <span class="comment">//!       @param[out] detector_data</span>
<a name="l01236"></a>01236 <span class="comment">//!       @param[out] image</span>
<a name="l01237"></a>01237 <span class="comment">//!       @param[out] source_data</span>
<a name="l01238"></a>01238 <span class="comment">//!       @param[out] file_name_voxels</span>
<a name="l01239"></a>01239 <span class="comment">//!       @param[out] file_name_materials</span>
<a name="l01240"></a>01240 <span class="comment">//!       @param[out] file_name_output</span>
<a name="l01241"></a>01241 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01242"></a><a class="code" href="MC-GPU__v1_83_8h.html#ad3f14826495baa5f44c9f830572c4bcd">01242</a> <span class="comment"></span><span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a04125b2129832959744717839c968ebb" title="Read the input file given in the command line and return the significant data.">read_input</a>(<span class="keywordtype">int</span> argc, <span class="keywordtype">char</span>** argv, <span class="keywordtype">int</span> myID, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>* total_histories, <span class="keywordtype">int</span>* seed_input, <span class="keywordtype">int</span>* gpu_id, <span class="keywordtype">int</span>* num_threads_per_block, <span class="keywordtype">int</span>* histories_per_thread, <span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>* detector_data, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>** image_ptr, <span class="keywordtype">int</span>* image_bytes, <span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>* source_data, <span class="keyword">struct</span> <a class="code" href="structsource__energy__struct.html" title="Structure storing the source energy spectrum data to be sampled using the Walker aliasing algorithm...">source_energy_struct</a>* source_energy_data, <span class="keywordtype">char</span>* file_name_voxels, <span class="keywordtype">char</span> file_name_materials[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>][250] , <span class="keywordtype">char</span>* file_name_output, <span class="keywordtype">char</span>* file_name_espc, <span class="keywordtype">int</span>* num_projections, <span class="keywordtype">double</span>* D_angle, <span class="keywordtype">double</span>* angularROI_0, <span class="keywordtype">double</span>* angularROI_1, <span class="keywordtype">double</span>* initial_angle, <a class="code" href="structulonglong2.html">ulonglong2</a>** voxels_Edep_ptr, <span class="keywordtype">int</span>* voxels_Edep_bytes, <span class="keywordtype">char</span>* file_dose_output, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_x_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_x_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_y_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_y_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_z_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_z_max, <span class="keywordtype">double</span>* SRotAxisD, <span class="keywordtype">double</span>* vertical_translation_per_projection, <span class="keywordtype">int</span>* flag_material_dose)
<a name="l01243"></a>01243 {
<a name="l01244"></a>01244   FILE* file_ptr = NULL;
<a name="l01245"></a>01245   <span class="keywordtype">char</span> new_line[250];
<a name="l01246"></a>01246   <span class="keywordtype">char</span> *new_line_ptr = NULL;
<a name="l01247"></a>01247   <span class="keywordtype">double</span> dummy_double;
<a name="l01248"></a>01248 
<a name="l01249"></a>01249   <span class="comment">// -- Read the input file name from command line, if given (otherwise keep default value):</span>
<a name="l01250"></a>01250   <span class="keywordflow">if</span> (2==argc)
<a name="l01251"></a>01251   {
<a name="l01252"></a>01252     file_ptr = fopen(argv[1], <span class="stringliteral">&quot;r&quot;</span>);
<a name="l01253"></a>01253     <span class="keywordflow">if</span> (NULL==file_ptr)
<a name="l01254"></a>01254     {
<a name="l01255"></a>01255       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file not found or not readable. Input file name: \&#39;%s\&#39;\n\n&quot;</span>, argv[1]);      
<a name="l01256"></a>01256         <span class="comment">//  Not finalizing MPI here because we want the execution to fail if there is a problem with any MPI thread!!! MPI_Finalize();   // Finalize MPI library: no more MPI calls allowed below.</span>
<a name="l01257"></a>01257       exit(-1);
<a name="l01258"></a>01258     }
<a name="l01259"></a>01259   }
<a name="l01260"></a>01260   <span class="keywordflow">else</span> <span class="keywordflow">if</span> (argc&gt;2)
<a name="l01261"></a>01261   {
<a name="l01262"></a>01262     
<a name="l01263"></a>01263     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Too many input parameter (argc=%d)!! Provide only the input file name.\n\n&quot;</span>, argc);    
<a name="l01264"></a>01264     <span class="comment">// Finalizing MPI because all threads will detect the same problem and fail together.</span>
<a name="l01265"></a>01265 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01266"></a>01266 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01267"></a>01267 <span class="preprocessor">    #endif</span>
<a name="l01268"></a>01268 <span class="preprocessor"></span>    exit(-1);
<a name="l01269"></a>01269   }
<a name="l01270"></a>01270   <span class="keywordflow">else</span>
<a name="l01271"></a>01271   {
<a name="l01272"></a>01272     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file name not given as an execution parameter!! Try again...\n\n&quot;</span>);
<a name="l01273"></a>01273 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01274"></a>01274 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01275"></a>01275 <span class="preprocessor">    #endif</span>
<a name="l01276"></a>01276 <span class="preprocessor"></span>    exit(-1);
<a name="l01277"></a>01277   }
<a name="l01278"></a>01278 
<a name="l01279"></a>01279   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n    -- Reading the input file \&#39;%s\&#39;:\n&quot;</span>, argv[1]);
<a name="l01280"></a>01280 
<a name="l01281"></a>01281 
<a name="l01282"></a>01282   <span class="comment">// -- Init. [SECTION SIMULATION CONFIG v.2009-05-12]:</span>
<a name="l01283"></a>01283   <span class="keywordflow">do</span>
<a name="l01284"></a>01284   {
<a name="l01285"></a>01285     new_line_ptr = fgets(new_line, 250, file_ptr);    <span class="comment">// Read full line (max. 250 characters).</span>
<a name="l01286"></a>01286     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01287"></a>01287     {
<a name="l01288"></a>01288       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION SIMULATION CONFIG v.2009-05-12\&#39;!!\n&quot;</span>);
<a name="l01289"></a>01289       exit(-2);
<a name="l01290"></a>01290     }
<a name="l01291"></a>01291   }
<a name="l01292"></a>01292   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION SIMULATION CONFIG v.2009-05-12&quot;</span>)==NULL);   <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01293"></a>01293   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01294"></a>01294     sscanf(new_line, <span class="stringliteral">&quot;%lf&quot;</span>, &amp;dummy_double);
<a name="l01295"></a>01295     *total_histories = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int) (dummy_double+0.0001);  <span class="comment">// Maximum unsigned long long value: 18446744073709551615</span>
<a name="l01296"></a>01296   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01297"></a>01297     sscanf(new_line, <span class="stringliteral">&quot;%d&quot;</span>, seed_input);   <span class="comment">// Set the RANECU PRNG seed (the same seed will be used to init the 2 MLCGs in RANECU)</span>
<a name="l01298"></a>01298   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01299"></a>01299     sscanf(new_line, <span class="stringliteral">&quot;%d&quot;</span>, gpu_id);       <span class="comment">// GPU NUMBER WHERE SIMULATION WILL RUN</span>
<a name="l01300"></a>01300   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01301"></a>01301     sscanf(new_line, <span class="stringliteral">&quot;%d&quot;</span>, num_threads_per_block);  <span class="comment">// GPU THREADS PER CUDA BLOCK</span>
<a name="l01302"></a>01302     
<a name="l01303"></a>01303 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l01304"></a>01304 <span class="preprocessor"></span>  <span class="keywordflow">if</span> ((*num_threads_per_block%32)!=0)
<a name="l01305"></a>01305   {
<a name="l01306"></a>01306     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! The input number of GPU threads per CUDA block must be a multiple of 32 (warp size). Input value: %d !!\n\n&quot;</span>, *num_threads_per_block);
<a name="l01307"></a>01307 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01308"></a>01308 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01309"></a>01309 <span class="preprocessor">    #endif</span>
<a name="l01310"></a>01310 <span class="preprocessor"></span>    exit(-2);
<a name="l01311"></a>01311   }
<a name="l01312"></a>01312 <span class="preprocessor">#endif</span>
<a name="l01313"></a>01313 <span class="preprocessor"></span>  new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01314"></a>01314     sscanf(new_line, <span class="stringliteral">&quot;%d&quot;</span>, histories_per_thread);   <span class="comment">// HISTORIES PER GPU THREAD</span>
<a name="l01315"></a>01315 
<a name="l01316"></a>01316     
<a name="l01317"></a>01317   <span class="comment">// -- Init. [SECTION SOURCE v.2009-05-12]:</span>
<a name="l01318"></a>01318   <span class="keywordflow">do</span>
<a name="l01319"></a>01319   {
<a name="l01320"></a>01320     new_line_ptr = fgets(new_line, 250, file_ptr);
<a name="l01321"></a>01321     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01322"></a>01322     {
<a name="l01323"></a>01323       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION SOURCE v.2011-07-12\&#39;!!\n&quot;</span>);
<a name="l01324"></a>01324       exit(-2);
<a name="l01325"></a>01325     }
<a name="l01326"></a>01326   }
<a name="l01327"></a>01327   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION SOURCE v.2011-07-12&quot;</span>)==NULL);   <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01328"></a>01328 
<a name="l01329"></a>01329 
<a name="l01330"></a>01330 
<a name="l01331"></a>01331     
<a name="l01332"></a>01332   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);  <span class="comment">// X-RAY ENERGY SPECTRUM FILE</span>
<a name="l01333"></a>01333     <a class="code" href="MC-GPU__v1_83_8cu.html#a96dba25b308814b25fbae71e27617f12" title="Extract a file name from an input text line, trimming the initial blanks, trailing comment (#) and st...">trim_name</a>(new_line, file_name_espc);
<a name="l01334"></a>01334     
<a name="l01335"></a>01335   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01336"></a>01336     sscanf(new_line, <span class="stringliteral">&quot;%f %f %f&quot;</span>, &amp;source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>, &amp;source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, &amp;source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);   <span class="comment">// SOURCE POSITION: X Y Z [cm]</span>
<a name="l01337"></a>01337   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01338"></a>01338     sscanf(new_line, <span class="stringliteral">&quot;%f %f %f&quot;</span>, &amp;source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>, &amp;source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, &amp;source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);   <span class="comment">// SOURCE DIRECTION COSINES: U V W</span>
<a name="l01339"></a>01339     <span class="comment">// -- Normalize the input beam direction to 1:</span>
<a name="l01340"></a>01340     dummy_double = 1.0/sqrt((<span class="keywordtype">double</span>)(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>));
<a name="l01341"></a>01341     source_data[0].direction.x = (float)(((<span class="keywordtype">double</span>)source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>)*dummy_double);
<a name="l01342"></a>01342     source_data[0].direction.y = (float)(((<span class="keywordtype">double</span>)source_data[0].direction.y)*dummy_double);
<a name="l01343"></a>01343     source_data[0].direction.z = (float)(((<span class="keywordtype">double</span>)source_data[0].direction.z)*dummy_double);
<a name="l01344"></a>01344   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01345"></a>01345   
<a name="l01346"></a>01346 
<a name="l01347"></a>01347   <span class="comment">// Read input fan beam polar (theta) and azimuthal (phi) aperture angles (deg):</span>
<a name="l01348"></a>01348   <span class="keywordtype">double</span> phi_aperture, theta_aperture;
<a name="l01349"></a>01349   sscanf(new_line, <span class="stringliteral">&quot;%lf %lf&quot;</span>, &amp;phi_aperture, &amp;theta_aperture);
<a name="l01350"></a>01350 
<a name="l01351"></a>01351   <span class="keywordflow">if</span> (theta_aperture &gt; 180.0)
<a name="l01352"></a>01352   {
<a name="l01353"></a>01353     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input polar aperture must be in [0,180] deg.!\n&quot;</span>);
<a name="l01354"></a>01354     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                       theta_aperture = %lf, phi_aperture = %lf\n&quot;</span>, theta_aperture, phi_aperture);
<a name="l01355"></a>01355 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01356"></a>01356 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01357"></a>01357 <span class="preprocessor">    #endif</span>
<a name="l01358"></a>01358 <span class="preprocessor"></span>    exit(-2);
<a name="l01359"></a>01359   }
<a name="l01360"></a>01360   <span class="keywordflow">if</span> (phi_aperture &gt; 360.0)
<a name="l01361"></a>01361   {
<a name="l01362"></a>01362     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input azimuthal aperture must be in [0,360] deg.!\n&quot;</span>);
<a name="l01363"></a>01363     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                       theta_aperture = %lf, phi_aperture = %lf\n&quot;</span>, theta_aperture, phi_aperture);
<a name="l01364"></a>01364 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01365"></a>01365 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01366"></a>01366 <span class="preprocessor">    #endif    </span>
<a name="l01367"></a>01367 <span class="preprocessor"></span>    exit(-2);
<a name="l01368"></a>01368   }
<a name="l01369"></a>01369   
<a name="l01370"></a>01370   <span class="comment">// Entering a negative theta_aperture or phi_aperture, the emitted fan beam will cover exactly the detector: see below.</span>
<a name="l01371"></a>01371     
<a name="l01372"></a>01372   <span class="comment">// *** RECTANGULAR BEAM INITIALIZATION: aperture initially centered at (0,1,0), ie, THETA_0=90, PHI_0=90</span>
<a name="l01373"></a>01373   <span class="comment">//     Using the algorithm used in PENMAIN.f, from penelope 2008 (by F. Salvat).</span>
<a name="l01374"></a>01374   source_data[0].cos_theta_low = (float)( cos((90.0 - 0.5*theta_aperture)*<a class="code" href="MC-GPU__v1_83_8h.html#af7e8592d0a634bd3642e9fd508ea8022">DEG2RAD</a>) );
<a name="l01375"></a>01375   source_data[0].D_cos_theta   = (float)( -2.0*source_data[0].<a class="code" href="structsource__struct.html#ad3b7a93633ec87d036453e54e7e7abad">cos_theta_low</a> );      <span class="comment">// Theta aperture is symetric above and below 90 deg</span>
<a name="l01376"></a>01376   source_data[0].phi_low       = (float)( (90.0 - 0.5*phi_aperture)*<a class="code" href="MC-GPU__v1_83_8h.html#af7e8592d0a634bd3642e9fd508ea8022">DEG2RAD</a> );
<a name="l01377"></a>01377   source_data[0].D_phi         = (float)( phi_aperture*DEG2RAD );
<a name="l01378"></a>01378   source_data[0].max_height_at_y1cm = (float) ( tan(0.5*theta_aperture*DEG2RAD) );
<a name="l01379"></a>01379 
<a name="l01380"></a>01380   
<a name="l01381"></a>01381   <span class="comment">// If a pencil beam is input, convert the 0 angle to a very small square beam to avoid precission errors:</span>
<a name="l01382"></a>01382   <span class="keywordflow">if</span> (abs(theta_aperture) &lt; 1.0e-7)
<a name="l01383"></a>01383   {
<a name="l01384"></a>01384     theta_aperture = +1.00e-7;
<a name="l01385"></a>01385     source_data[0].cos_theta_low = 0.0f;  <span class="comment">// = cos(90-0)</span>
<a name="l01386"></a>01386     source_data[0].D_cos_theta   = 0.0f;
<a name="l01387"></a>01387     source_data[0].max_height_at_y1cm = 0.0f;
<a name="l01388"></a>01388   }
<a name="l01389"></a>01389   <span class="keywordflow">if</span> (abs(phi_aperture) &lt; 1.0e-7)
<a name="l01390"></a>01390   {  
<a name="l01391"></a>01391     phi_aperture = +1.00e-7;
<a name="l01392"></a>01392     source_data[0].phi_low       = (float)( 90.0*DEG2RAD );
<a name="l01393"></a>01393     source_data[0].D_phi         = 0.0f;    
<a name="l01394"></a>01394   }  
<a name="l01395"></a>01395   
<a name="l01396"></a>01396   
<a name="l01397"></a>01397   <span class="comment">// -- Init. [SECTION IMAGE DETECTOR v.2009-12-02]:</span>
<a name="l01398"></a>01398   <span class="keywordflow">do</span>
<a name="l01399"></a>01399   {
<a name="l01400"></a>01400     new_line_ptr = fgets(new_line, 250, file_ptr);
<a name="l01401"></a>01401     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01402"></a>01402     {
<a name="l01403"></a>01403       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION IMAGE DETECTOR v.2009-12-02\&#39;!!\n&quot;</span>);
<a name="l01404"></a>01404       exit(-2);
<a name="l01405"></a>01405     }
<a name="l01406"></a>01406   }
<a name="l01407"></a>01407   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION IMAGE DETECTOR v.2009-12-02&quot;</span>)==NULL);   <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01408"></a>01408   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01409"></a>01409     <a class="code" href="MC-GPU__v1_83_8cu.html#a96dba25b308814b25fbae71e27617f12" title="Extract a file name from an input text line, trimming the initial blanks, trailing comment (#) and st...">trim_name</a>(new_line, file_name_output);   <span class="comment">// OUTPUT IMAGE FILE NAME (no spaces)</span>
<a name="l01410"></a>01410   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01411"></a>01411     <span class="keywordtype">float</span> dummy_num_pixels_x, dummy_num_pixels_y;  <span class="comment">// Read input pixel number as float and truncated to integer</span>
<a name="l01412"></a>01412     sscanf(new_line, <span class="stringliteral">&quot;%f %f&quot;</span>, &amp;dummy_num_pixels_x, &amp;dummy_num_pixels_y);   <span class="comment">// NUMBER OF PIXELS IN THE IMAGE: Nx Nz</span>
<a name="l01413"></a>01413     detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a> = (int)(dummy_num_pixels_x+0.001f);
<a name="l01414"></a>01414     detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a> = (int)(dummy_num_pixels_y+0.001f);
<a name="l01415"></a>01415     detector_data[0].<a class="code" href="structdetector__struct.html#ac26d0b66ebf521346d8ba07609340ea6">total_num_pixels</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a> * detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>;
<a name="l01416"></a>01416 
<a name="l01417"></a>01417     <span class="keywordflow">if</span> (detector_data[0].total_num_pixels &lt; 1 || detector_data[0].total_num_pixels &gt; 99999999 )
<a name="l01418"></a>01418     {
<a name="l01419"></a>01419       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! The input number of pixels is incorrect. Input: X_pix = %d, Y_pix = %d, total_num_pix = %d!!\n\n&quot;</span>, detector_data[0].num_pixels.x, detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>, detector_data[0].<a class="code" href="structdetector__struct.html#ac26d0b66ebf521346d8ba07609340ea6">total_num_pixels</a>);
<a name="l01420"></a>01420 <span class="preprocessor">      #ifdef USING_MPI</span>
<a name="l01421"></a>01421 <span class="preprocessor"></span>        MPI_Finalize();
<a name="l01422"></a>01422 <span class="preprocessor">      #endif      </span>
<a name="l01423"></a>01423 <span class="preprocessor"></span>      exit(-2); 
<a name="l01424"></a>01424     }
<a name="l01425"></a>01425   
<a name="l01426"></a>01426   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01427"></a>01427   sscanf(new_line, <span class="stringliteral">&quot;%f %f&quot;</span>, &amp;detector_data[0].width_X, &amp;detector_data[0].height_Z);   <span class="comment">// IMAGE SIZE (width, height): Dx Dz [cm]</span>
<a name="l01428"></a>01428     detector_data[0].<a class="code" href="structdetector__struct.html#a3c6f9e59e89db06211ccea63287e2826">inv_pixel_size_X</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a> / detector_data[0].<a class="code" href="structdetector__struct.html#a1f67c512205953e75552865ae7a2490a">width_X</a>;
<a name="l01429"></a>01429     detector_data[0].<a class="code" href="structdetector__struct.html#aa91fc94f674ad7af8eca577f4d8cd22b">inv_pixel_size_Z</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a> / detector_data[0].<a class="code" href="structdetector__struct.html#af61592c77c6da548fff31215bf956dd8">height_Z</a>;
<a name="l01430"></a>01430 
<a name="l01431"></a>01431   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01432"></a>01432     sscanf(new_line, <span class="stringliteral">&quot;%f&quot;</span>, &amp;detector_data[0].sdd);            <span class="comment">// SOURCE-TO-DETECTOR DISTANCE [cm] (detector set in front of the source, normal to the input direction)</span>
<a name="l01433"></a>01433 
<a name="l01434"></a>01434     <a class="code" href="structfloat3.html">float3</a> detector_center;   <span class="comment">// Center of the detector straight ahead of the focal spot.</span>
<a name="l01435"></a>01435     detector_center.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = source_data[0].position.x + source_data[0].direction.x * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l01436"></a>01436     detector_center.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = source_data[0].position.y + source_data[0].direction.y * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l01437"></a>01437     detector_center.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = source_data[0].position.z + source_data[0].direction.z * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l01438"></a>01438         
<a name="l01439"></a>01439     <span class="keywordflow">if</span> ((detector_data[0].sdd)&lt;1.0e-6)
<a name="l01440"></a>01440     {
<a name="l01441"></a>01441       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! The source-to-detector distance must be positive. Input: sdd=%f!!\n\n&quot;</span>, detector_data[0].sdd);
<a name="l01442"></a>01442 <span class="preprocessor">      #ifdef USING_MPI</span>
<a name="l01443"></a>01443 <span class="preprocessor"></span>        MPI_Finalize();
<a name="l01444"></a>01444 <span class="preprocessor">      #endif      </span>
<a name="l01445"></a>01445 <span class="preprocessor"></span>      exit(-2);
<a name="l01446"></a>01446     }
<a name="l01447"></a>01447 
<a name="l01448"></a>01448   <span class="keywordflow">if</span> ( (theta_aperture &lt; -1.0e-7) || (phi_aperture &lt; -1.0e-7) )   <span class="comment">// If we enter a negative angle, the fan beam will cover exactly the detector surface.</span>
<a name="l01449"></a>01449   {
<a name="l01450"></a>01450     theta_aperture= 2.0 * atan(0.5*detector_data[0].height_Z/(detector_data[0].sdd)) * <a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>;   <span class="comment">// Optimum angles</span>
<a name="l01451"></a>01451     phi_aperture  = 2.0 * atan(0.5*detector_data[0].width_X/(detector_data[0].sdd)) * <a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>;
<a name="l01452"></a>01452 
<a name="l01453"></a>01453     source_data[0].cos_theta_low = (float)( cos((90.0 - 0.5*theta_aperture)*DEG2RAD) );
<a name="l01454"></a>01454     source_data[0].D_cos_theta   = (float)( -2.0*source_data[0].cos_theta_low );      <span class="comment">// Theta aperture is symetric above and below 90 deg</span>
<a name="l01455"></a>01455     source_data[0].phi_low       = (float)( (90.0 - 0.5*phi_aperture)*<a class="code" href="MC-GPU__v1_83_8h.html#af7e8592d0a634bd3642e9fd508ea8022">DEG2RAD</a> );
<a name="l01456"></a>01456     source_data[0].D_phi         = (float)( phi_aperture*DEG2RAD );
<a name="l01457"></a>01457     source_data[0].max_height_at_y1cm = (float) ( tan(0.5*theta_aperture*DEG2RAD) ); 
<a name="l01458"></a>01458   }
<a name="l01459"></a>01459   
<a name="l01460"></a>01460 
<a name="l01461"></a>01461   <span class="comment">// -- Init. [SECTION CT SCAN v.2011-10-25]:</span>
<a name="l01462"></a>01462   <span class="keywordflow">do</span>
<a name="l01463"></a>01463   {
<a name="l01464"></a>01464     new_line_ptr = fgets(new_line, 250, file_ptr);
<a name="l01465"></a>01465     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01466"></a>01466     {
<a name="l01467"></a>01467       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION CT SCAN TRAJECTORY v.2011-10-25\&#39;!!\n&quot;</span>);
<a name="l01468"></a>01468       exit(-2);
<a name="l01469"></a>01469     }
<a name="l01470"></a>01470   }
<a name="l01471"></a>01471   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION CT SCAN TRAJECTORY v.2011-10-25&quot;</span>)==NULL);  <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01472"></a>01472   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01473"></a>01473   sscanf(new_line, <span class="stringliteral">&quot;%d&quot;</span>, num_projections);     <span class="comment">// NUMBER OF PROJECTIONS (beam must be perpendicular to Z axis, set to 1 for a single projection)</span>
<a name="l01474"></a>01474   <span class="keywordflow">if</span> (0 == (*num_projections))
<a name="l01475"></a>01475     *num_projections = 1;      <span class="comment">// Zero projections has the same effect as 1 projection (ie, no CT scan rotation). Negative values are allowed and then the source rotates in opposite direction (negative angles).</span>
<a name="l01476"></a>01476   <span class="keywordflow">if</span> ( (fabs(*num_projections) &gt; 1) &amp;&amp; (fabs(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>)&gt;0.00001f) )
<a name="l01477"></a>01477   {
<a name="l01478"></a>01478     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Sorry, but currently we can only simulate CT scans when the source direction is perpendicular to the Z axis (ie, w=0).\n\n\n&quot;</span>);   <span class="comment">// The reconstructed planes are always parallel to the XY plane.\n&quot;);</span>
<a name="l01479"></a>01479 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01480"></a>01480 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01481"></a>01481 <span class="preprocessor">    #endif    </span>
<a name="l01482"></a>01482 <span class="preprocessor"></span>    exit(-2);
<a name="l01483"></a>01483   }
<a name="l01484"></a>01484   <span class="keywordflow">if</span> ( fabs(*num_projections) &gt; <a class="code" href="MC-GPU__v1_83_8h.html#a30b26edf408e1ff6034eb712ab7e173e" title="Maximum number of projections allowed in the CT simulation (not limited by the constant memory becaus...">MAX_NUM_PROJECTIONS</a> )
<a name="l01485"></a>01485   {
<a name="l01486"></a>01486     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! The input number of projections is too large. Increase parameter MAX_NUM_PROJECTIONS=%d in the header file and recompile.\n&quot;</span>, <a class="code" href="MC-GPU__v1_83_8h.html#a30b26edf408e1ff6034eb712ab7e173e" title="Maximum number of projections allowed in the CT simulation (not limited by the constant memory becaus...">MAX_NUM_PROJECTIONS</a>);
<a name="l01487"></a>01487     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(    <span class="stringliteral">&quot;                        There is no limit in the number of projections to be simulated because the source, detector data for each projection is stored in global memory and transfered to shared memory for each projection.\n\n&quot;</span>);
<a name="l01488"></a>01488 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01489"></a>01489 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01490"></a>01490 <span class="preprocessor">    #endif    </span>
<a name="l01491"></a>01491 <span class="preprocessor"></span>    exit(-2);
<a name="l01492"></a>01492   }
<a name="l01493"></a>01493   
<a name="l01494"></a>01494 
<a name="l01495"></a>01495   <span class="keywordflow">if</span> (*num_projections!=1)
<a name="l01496"></a>01496   {
<a name="l01497"></a>01497     <span class="comment">// -- Skip rest of the section if simulating a single projection:</span>
<a name="l01498"></a>01498     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01499"></a>01499     sscanf(new_line, <span class="stringliteral">&quot;%lf&quot;</span>, D_angle);   <span class="comment">// ANGLE BETWEEN PROJECTIONS [degrees] (360/num_projections for full CT)</span>
<a name="l01500"></a>01500 <span class="comment">//     printf(&quot;       [Input] %s&quot;,new_line);</span>
<a name="l01501"></a>01501     *D_angle = (*D_angle)*<a class="code" href="MC-GPU__v1_83_8h.html#af7e8592d0a634bd3642e9fd508ea8022">DEG2RAD</a>;      <span class="comment">// store the angle in radians</span>
<a name="l01502"></a>01502     
<a name="l01503"></a>01503 
<a name="l01504"></a>01504     <span class="comment">// Calculate initial source angle:</span>
<a name="l01505"></a>01505     *initial_angle = acos((<span class="keywordtype">double</span>)(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>));
<a name="l01506"></a>01506     <span class="keywordflow">if</span> (source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>&lt;0)
<a name="l01507"></a>01507       *initial_angle = -(*initial_angle);     <span class="comment">// Correct for the fact that positive and negative angles have the same ACOS</span>
<a name="l01508"></a>01508     <span class="keywordflow">if</span> (*initial_angle&lt;0.0)
<a name="l01509"></a>01509       *initial_angle = (*initial_angle) + 2.0*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Make sure the angle is not negative, between [0,360) degrees.</span>
<a name="l01510"></a>01510     *initial_angle = (*initial_angle) - <a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Correct the fact that the source is opposite to the detector (180 degrees difference).</span>
<a name="l01511"></a>01511     <span class="keywordflow">if</span> (*initial_angle&lt;0.0)
<a name="l01512"></a>01512       *initial_angle = (*initial_angle) + 2.0*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Make sure the initial angle is not negative, between [0,360) degrees.</span>
<a name="l01513"></a>01513 
<a name="l01514"></a>01514   
<a name="l01515"></a>01515     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01516"></a>01516     sscanf(new_line, <span class="stringliteral">&quot;%lf %lf&quot;</span>, angularROI_0, angularROI_1);   <span class="comment">// ANGLES OF INTEREST (projections outside this interval will be skipped)</span>
<a name="l01517"></a>01517     
<a name="l01518"></a>01518     <span class="comment">// if (*angularROI_0&lt;-0.001 || *angularROI_1&gt;360.001)</span>
<a name="l01519"></a>01519     <span class="comment">// {</span>
<a name="l01520"></a>01520     <span class="comment">//   MASTER_THREAD printf(&quot;\n\n   !!read_input ERROR!! The angles in the angular region of interest must be in the interval [0,360]. Input: %f, %f.\n\n\n&quot;, *angularROI_0, *angularROI_1);   // // The reconstructed planes are always parallel to the XY plane.\n&quot;);</span>
<a name="l01521"></a>01521     <span class="comment">//   #ifdef USING_MPI</span>
<a name="l01522"></a>01522     <span class="comment">//     MPI_Finalize();</span>
<a name="l01523"></a>01523     <span class="comment">//   #endif</span>
<a name="l01524"></a>01524     <span class="comment">//   exit(-2);</span>
<a name="l01525"></a>01525     <span class="comment">// }</span>
<a name="l01526"></a>01526     
<a name="l01527"></a>01527     *angularROI_0 = (*angularROI_0 - 0.00001)*<a class="code" href="MC-GPU__v1_83_8h.html#af7e8592d0a634bd3642e9fd508ea8022">DEG2RAD</a>;   <span class="comment">// Store the angles of interest in radians, increasing a little the interval to avoid floating point precision problems</span>
<a name="l01528"></a>01528     *angularROI_1 = (*angularROI_1 + 0.00001)*<a class="code" href="MC-GPU__v1_83_8h.html#af7e8592d0a634bd3642e9fd508ea8022">DEG2RAD</a>;
<a name="l01529"></a>01529 
<a name="l01530"></a>01530 
<a name="l01531"></a>01531     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01532"></a>01532     sscanf(new_line, <span class="stringliteral">&quot;%lf&quot;</span>, SRotAxisD);   <span class="comment">// SOURCE-TO-ROTATION AXIS DISTANCE (rotation axis parallel to Z, located between the source and the detector)</span>
<a name="l01533"></a>01533     <span class="keywordflow">if</span> (*SRotAxisD&lt;0.0 || *SRotAxisD&gt;detector_data[0].sdd)
<a name="l01534"></a>01534     {
<a name="l01535"></a>01535       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Invalid source-to-rotation axis distance! Input: %f (sdd=%f).\n\n\n&quot;</span>, *SRotAxisD, detector_data[0].sdd);
<a name="l01536"></a>01536 <span class="preprocessor">      #ifdef USING_MPI</span>
<a name="l01537"></a>01537 <span class="preprocessor"></span>        MPI_Finalize();
<a name="l01538"></a>01538 <span class="preprocessor">      #endif      </span>
<a name="l01539"></a>01539 <span class="preprocessor"></span>      exit(-2);
<a name="l01540"></a>01540     }
<a name="l01541"></a>01541     
<a name="l01542"></a>01542     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01543"></a>01543     sscanf(new_line, <span class="stringliteral">&quot;%lf&quot;</span>, vertical_translation_per_projection);    <span class="comment">// VERTICAL TRANSLATION BETWEEN PROJECTIONS (HELICAL SCAN)        </span>
<a name="l01544"></a>01544     
<a name="l01545"></a>01545   }
<a name="l01546"></a>01546   
<a name="l01547"></a>01547   
<a name="l01548"></a>01548 
<a name="l01549"></a>01549   <span class="comment">// -- Init. [SECTION DOSE DEPOSITION v.2012-12-12] (MC-GPU v.1.3):</span>
<a name="l01550"></a>01550   <span class="comment">//    Electrons are not transported and therefore we are approximating that the dose is equal to the KERMA (energy released by the photons alone).</span>
<a name="l01551"></a>01551   <span class="comment">//    This approximation is acceptable when there is electronic equilibrium and when the range of the secondary electrons is shorter than the voxel size.</span>
<a name="l01552"></a>01552   <span class="comment">//    Usually the doses will be acceptable for photon energies below 1 MeV. The dose estimates may not be accurate at the interface of low density volumes.</span>
<a name="l01553"></a>01553   <span class="keywordflow">do</span>
<a name="l01554"></a>01554   {
<a name="l01555"></a>01555     new_line_ptr = fgets(new_line, 250, file_ptr);
<a name="l01556"></a>01556     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01557"></a>01557     {
<a name="l01558"></a>01558       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION DOSE DEPOSITION v.2012-12-12\&#39;!!\n&quot;</span>);
<a name="l01559"></a>01559       exit(-2);
<a name="l01560"></a>01560     }
<a name="l01561"></a>01561     
<a name="l01562"></a>01562     <span class="keywordflow">if</span> (strstr(new_line,<span class="stringliteral">&quot;SECTION DOSE DEPOSITION v.2011-02-18&quot;</span>)!=NULL)  <span class="comment">// Detect previous version of input file</span>
<a name="l01563"></a>01563     {
<a name="l01564"></a>01564       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Please update the input file to the new version of MC-GPU (v1.3)!!\n\n    You simply have to change the input file text line:\n         [SECTION DOSE DEPOSITION v.2011-02-18]\n\n    for these two lines:\n         [SECTION DOSE DEPOSITION v.2012-12-12]\n         NO                              # TALLY MATERIAL DOSE? [YES/NO]\n\n&quot;</span>);
<a name="l01565"></a>01565       exit(-2);
<a name="l01566"></a>01566     }
<a name="l01567"></a>01567     
<a name="l01568"></a>01568   }
<a name="l01569"></a>01569   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION DOSE DEPOSITION v.2012-12-12&quot;</span>)==NULL);  <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01570"></a>01570     
<a name="l01571"></a>01571 
<a name="l01572"></a>01572   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);   <span class="comment">// TALLY MATERIAL DOSE? [YES/NO]  --&gt; turn on/off the material dose tallied adding the Edep in each material, independently of the voxels.</span>
<a name="l01573"></a>01573   <span class="keywordflow">if</span> (0==strncmp(<span class="stringliteral">&quot;YE&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;Ye&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;ye&quot;</span>,new_line,2))
<a name="l01574"></a>01574   {
<a name="l01575"></a>01575     *flag_material_dose = 1;
<a name="l01576"></a>01576     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Material dose deposition tally ENABLED.\n&quot;</span>);
<a name="l01577"></a>01577   }
<a name="l01578"></a>01578   <span class="keywordflow">else</span> <span class="keywordflow">if</span> (0==strncmp(<span class="stringliteral">&quot;NO&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;No&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;no&quot;</span>,new_line,2))
<a name="l01579"></a>01579   {
<a name="l01580"></a>01580     *flag_material_dose = 0;  <span class="comment">// -- NO: disabling tally</span>
<a name="l01581"></a>01581     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Material dose deposition tally DISABLED.\n&quot;</span>);    
<a name="l01582"></a>01582   }
<a name="l01583"></a>01583   <span class="keywordflow">else</span>
<a name="l01584"></a>01584   {
<a name="l01585"></a>01585     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Answer YES or NO in the first two line of \&#39;SECTION DOSE DEPOSITION\&#39; to enable or disable the material dose and 3D voxel dose tallies.\n                        Input text: %s\n\n&quot;</span>,new_line);
<a name="l01586"></a>01586 <span class="preprocessor">    #ifdef USING_MPI</span>
<a name="l01587"></a>01587 <span class="preprocessor"></span>      MPI_Finalize();
<a name="l01588"></a>01588 <span class="preprocessor">    #endif</span>
<a name="l01589"></a>01589 <span class="preprocessor"></span>    exit(-2);
<a name="l01590"></a>01590   }        
<a name="l01591"></a>01591 
<a name="l01592"></a>01592   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);   <span class="comment">// TALLY 3D VOXEL DOSE? [YES/NO] </span>
<a name="l01593"></a>01593 
<a name="l01594"></a>01594   <span class="keywordflow">if</span> (0==strncmp(<span class="stringliteral">&quot;YE&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;Ye&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;ye&quot;</span>,new_line,2))
<a name="l01595"></a>01595   {
<a name="l01596"></a>01596     <span class="comment">// -- YES: using the tally</span>
<a name="l01597"></a>01597     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr); <a class="code" href="MC-GPU__v1_83_8cu.html#a96dba25b308814b25fbae71e27617f12" title="Extract a file name from an input text line, trimming the initial blanks, trailing comment (#) and st...">trim_name</a>(new_line, file_dose_output);   <span class="comment">// OUTPUT DOSE FILE NAME (no spaces)</span>
<a name="l01598"></a>01598     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr); sscanf(new_line, <span class="stringliteral">&quot;%hd %hd&quot;</span>, dose_ROI_x_min, dose_ROI_x_max);   <span class="comment">// # VOXELS TO TALLY DOSE: X-index min max (first voxel has index 1)</span>
<a name="l01599"></a>01599     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr); sscanf(new_line, <span class="stringliteral">&quot;%hd %hd&quot;</span>, dose_ROI_y_min, dose_ROI_y_max);   <span class="comment">// # VOXELS TO TALLY DOSE: Y-index min max</span>
<a name="l01600"></a>01600     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr); sscanf(new_line, <span class="stringliteral">&quot;%hd %hd&quot;</span>, dose_ROI_z_min, dose_ROI_z_max);   <span class="comment">// # VOXELS TO TALLY DOSE: Z-index min max</span>
<a name="l01601"></a>01601 
<a name="l01602"></a>01602     *dose_ROI_x_min -= 1; *dose_ROI_x_max -= 1;  <span class="comment">// -Re-scale input coordinates to have index=0 for the first voxel instead of 1.</span>
<a name="l01603"></a>01603     *dose_ROI_y_min -= 1; *dose_ROI_y_max -= 1;
<a name="l01604"></a>01604     *dose_ROI_z_min -= 1; *dose_ROI_z_max -= 1;
<a name="l01605"></a>01605 
<a name="l01606"></a>01606     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       3D voxel dose deposition tally ENABLED.\n&quot;</span>);
<a name="l01607"></a>01607     <span class="keywordflow">if</span> ( ((*dose_ROI_x_min)&gt;(*dose_ROI_x_max)) || ((*dose_ROI_y_min)&gt;(*dose_ROI_y_max)) || ((*dose_ROI_z_min)&gt;(*dose_ROI_z_max)) ||
<a name="l01608"></a>01608           (*dose_ROI_x_min)&lt;0 || (*dose_ROI_y_min)&lt;0 || (*dose_ROI_z_min)&lt;0 )
<a name="l01609"></a>01609     {
<a name="l01610"></a>01610       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! The input region-of-interst in \&#39;SECTION DOSE DEPOSITION\&#39; is not valid: the minimum voxel index may not be zero or larger than the maximum index.\n&quot;</span>);
<a name="l01611"></a>01611       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(  <span class="stringliteral">&quot;                          Input data = X[%d,%d], Y[%d,%d], Z[%d,%d]\n\n&quot;</span>, *dose_ROI_x_min+1, *dose_ROI_x_max+1, *dose_ROI_y_min+1, *dose_ROI_y_max+1, *dose_ROI_z_min+1, *dose_ROI_z_max+1);  <span class="comment">// Show ROI with index=1 for the first voxel instead of 0.</span>
<a name="l01612"></a>01612 <span class="preprocessor">      #ifdef USING_MPI</span>
<a name="l01613"></a>01613 <span class="preprocessor"></span>        MPI_Finalize();
<a name="l01614"></a>01614 <span class="preprocessor">      #endif      </span>
<a name="l01615"></a>01615 <span class="preprocessor"></span>      exit(-2);
<a name="l01616"></a>01616     }
<a name="l01617"></a>01617     <span class="keywordflow">if</span> ( ((*dose_ROI_x_min)==(*dose_ROI_x_max)) &amp;&amp; ((*dose_ROI_y_min)==(*dose_ROI_y_max)) &amp;&amp; ((*dose_ROI_z_min)==(*dose_ROI_z_max)) ) 
<a name="l01618"></a>01618     {
<a name="l01619"></a>01619       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input!! According to the input region-of-interest in \&#39;SECTION DOSE DEPOSITION\&#39;, only the dose in the voxel (%d,%d,%d) will be tallied.\n\n&quot;</span>,*dose_ROI_x_min,*dose_ROI_y_min,*dose_ROI_z_min);
<a name="l01620"></a>01620     }
<a name="l01621"></a>01621     
<a name="l01622"></a>01622   }
<a name="l01623"></a>01623   <span class="keywordflow">else</span> <span class="keywordflow">if</span> (0==strncmp(<span class="stringliteral">&quot;NO&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;No&quot;</span>,new_line,2) || 0==strncmp(<span class="stringliteral">&quot;no&quot;</span>,new_line,2))
<a name="l01624"></a>01624   {
<a name="l01625"></a>01625     <span class="comment">// -- NO: disabling tally</span>
<a name="l01626"></a>01626     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       3D voxel dose deposition tally DISABLED.\n&quot;</span>);
<a name="l01627"></a>01627     *dose_ROI_x_min = (<span class="keywordtype">short</span> int) 32500; *dose_ROI_x_max = (<span class="keywordtype">short</span> int) -32500;   <span class="comment">// Set absurd values for the ROI to make sure we never get any dose tallied</span>
<a name="l01628"></a>01628     *dose_ROI_y_min = (<span class="keywordtype">short</span> int) 32500; *dose_ROI_y_max = (<span class="keywordtype">short</span> int) -32500;   <span class="comment">// (the maximum values for short int variables are +-32768)</span>
<a name="l01629"></a>01629     *dose_ROI_z_min = (<span class="keywordtype">short</span> int) 32500; *dose_ROI_z_max = (<span class="keywordtype">short</span> int) -32500;
<a name="l01630"></a>01630   }
<a name="l01631"></a>01631   <span class="keywordflow">else</span>
<a name="l01632"></a>01632   {
<a name="l01633"></a>01633       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Answer YES or NO in the first two line of \&#39;SECTION DOSE DEPOSITION\&#39; to enable or disable the material dose and 3D voxel dose tallies.\n                        Input text: %s\n\n&quot;</span>,new_line);
<a name="l01634"></a>01634 <span class="preprocessor">      #ifdef USING_MPI</span>
<a name="l01635"></a>01635 <span class="preprocessor"></span>        MPI_Finalize();
<a name="l01636"></a>01636 <span class="preprocessor">      #endif</span>
<a name="l01637"></a>01637 <span class="preprocessor"></span>      exit(-2);
<a name="l01638"></a>01638   }
<a name="l01639"></a>01639   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n&quot;</span>);
<a name="l01640"></a>01640 
<a name="l01641"></a>01641 
<a name="l01642"></a>01642 
<a name="l01643"></a>01643   <span class="comment">// -- Init. [SECTION VOXELIZED GEOMETRY FILE v.2009-11-30]:</span>
<a name="l01644"></a>01644   <span class="keywordflow">do</span>
<a name="l01645"></a>01645   {
<a name="l01646"></a>01646     new_line_ptr = fgets(new_line, 250, file_ptr);
<a name="l01647"></a>01647     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01648"></a>01648     {
<a name="l01649"></a>01649       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION VOXELIZED GEOMETRY FILE v.2009-11-30\&#39;!!\n&quot;</span>);
<a name="l01650"></a>01650       exit(-2);
<a name="l01651"></a>01651     }
<a name="l01652"></a>01652   }
<a name="l01653"></a>01653   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION VOXELIZED GEOMETRY FILE v.2009-11-30&quot;</span>)==NULL);   <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01654"></a>01654   new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01655"></a>01655   <a class="code" href="MC-GPU__v1_83_8cu.html#a96dba25b308814b25fbae71e27617f12" title="Extract a file name from an input text line, trimming the initial blanks, trailing comment (#) and st...">trim_name</a>(new_line, file_name_voxels);   <span class="comment">// VOXEL GEOMETRY FILE (penEasy 2008 format)</span>
<a name="l01656"></a>01656 
<a name="l01657"></a>01657   <span class="keywordflow">do</span>
<a name="l01658"></a>01658   {
<a name="l01659"></a>01659     new_line_ptr = fgets(new_line, 250, file_ptr);
<a name="l01660"></a>01660     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01661"></a>01661     {
<a name="l01662"></a>01662       printf(<span class="stringliteral">&quot;\n\n   !!read_input ERROR!! Input file is not readable or does not contain the string \&#39;SECTION MATERIAL FILE LIST\&#39;!!\n&quot;</span>);
<a name="l01663"></a>01663       exit(-2);
<a name="l01664"></a>01664     }
<a name="l01665"></a>01665   }
<a name="l01666"></a>01666   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;SECTION MATERIAL&quot;</span>)==NULL);   <span class="comment">// Skip comments and empty lines until the section begins</span>
<a name="l01667"></a>01667 
<a name="l01668"></a>01668   <span class="keywordtype">int</span> i;
<a name="l01669"></a>01669   <span class="keywordflow">for</span> (i=0; i&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; i++)
<a name="l01670"></a>01670   {
<a name="l01671"></a>01671     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);
<a name="l01672"></a>01672     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01673"></a>01673       file_name_materials[i][0]=<span class="charliteral">&#39;\n&#39;</span>;   <span class="comment">// The input file is allowed to finish without defining all the materials</span>
<a name="l01674"></a>01674     <span class="keywordflow">else</span>
<a name="l01675"></a>01675       <a class="code" href="MC-GPU__v1_83_8cu.html#a96dba25b308814b25fbae71e27617f12" title="Extract a file name from an input text line, trimming the initial blanks, trailing comment (#) and st...">trim_name</a>(new_line, file_name_materials[i]);
<a name="l01676"></a>01676   }
<a name="l01677"></a>01677   <span class="comment">// [Finish reading input file]</span>
<a name="l01678"></a>01678 <span class="comment"></span>
<a name="l01679"></a>01679 <span class="comment">  /////////////////////////////////////////////////////////////////////////////</span>
<a name="l01680"></a>01680 <span class="comment"></span>
<a name="l01681"></a>01681   <span class="comment">// *** Set the rotation that will bring particles from the detector plane to +Y=(0,+1,0) through a rotation around X and around Z (counter-clock):</span>
<a name="l01682"></a>01682   <span class="keywordtype">double</span> rotX, rotZ, cos_rX, cos_rZ, sin_rX, sin_rZ;
<a name="l01683"></a>01683       <span class="comment">// rotX = 1.5*PI - acos(source_data[0].direction.z);  // Rotate to +Y = (0,+1,0) --&gt; rotX_0 = 3/2*PI == -PI/2</span>
<a name="l01684"></a>01684     rotX = acos(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>) - 0.5*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;  <span class="comment">// Rotate to +Y = (0,+1,0) --&gt; rotX_0 =  -PI/2</span>
<a name="l01685"></a>01685       <span class="comment">// rotX = 0.5*PI - acos(source_data[0].direction.z);  // Rotate to +Y = (0,+1,0) --&gt; rotX_0 =  PI/2</span>
<a name="l01686"></a>01686   <span class="keywordflow">if</span> ( (source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*source_data[0].direction.x + source_data[0].direction.y*source_data[0].direction.y) &gt; 1.0e-8 )   <span class="comment">// == u^2+v^2 &gt; 0</span>
<a name="l01687"></a>01687   {
<a name="l01688"></a>01688       <span class="comment">// rotZ = 0.5*PI - acos(source_data[0].direction.x/sqrt(source_data[0].direction.x*source_data[0].direction.x + source_data[0].direction.y*source_data[0].direction.y));</span>
<a name="l01689"></a>01689     <span class="keywordflow">if</span> (source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> &gt;= 0.0f)
<a name="l01690"></a>01690       rotZ = 0.5*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a> - acos(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>/sqrt(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*source_data[0].direction.x + source_data[0].direction.y*source_data[0].direction.y));
<a name="l01691"></a>01691     <span class="keywordflow">else</span>
<a name="l01692"></a>01692       rotZ = 0.5*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a> - (-acos(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>/sqrt(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*source_data[0].direction.x + source_data[0].direction.y*source_data[0].direction.y)));
<a name="l01693"></a>01693   }
<a name="l01694"></a>01694   <span class="keywordflow">else</span>
<a name="l01695"></a>01695     rotZ = 0.0;   <span class="comment">// Vector pointing to +Z, do not rotate around Z then.</span>
<a name="l01696"></a>01696  
<a name="l01697"></a>01697   <span class="comment">// -- Set the rotation matrix RzRx (called inverse because moves from the correct position to the reference at +Y):</span>
<a name="l01698"></a>01698   cos_rX = cos(rotX);
<a name="l01699"></a>01699   cos_rZ = cos(rotZ);
<a name="l01700"></a>01700   sin_rX = sin(rotX);
<a name="l01701"></a>01701   sin_rZ = sin(rotZ);
<a name="l01702"></a>01702 
<a name="l01703"></a>01703   <span class="comment">// Rotation matrix RxRz:</span>
<a name="l01704"></a>01704   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[0] =  cos_rZ;
<a name="l01705"></a>01705   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[1] = -sin_rZ;
<a name="l01706"></a>01706   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[2] =  0.0f;
<a name="l01707"></a>01707   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[3] =  cos_rX*sin_rZ;
<a name="l01708"></a>01708   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[4] =  cos_rX*cos_rZ;
<a name="l01709"></a>01709   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[5] = -sin_rX;
<a name="l01710"></a>01710   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[6] =  sin_rX*sin_rZ;
<a name="l01711"></a>01711   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[7] =  sin_rX*cos_rZ;
<a name="l01712"></a>01712   detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[8] =  cos_rX;
<a name="l01713"></a>01713 
<a name="l01714"></a>01714 
<a name="l01715"></a>01715 
<a name="l01716"></a>01716   <span class="keywordflow">if</span> ((source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> &gt; 0.99999f) &amp;&amp; (*num_projections==1))
<a name="l01717"></a>01717   {
<a name="l01718"></a>01718     <span class="comment">// Simulating a single projection and initial beam pointing to +Y: no rotation needed!!</span>
<a name="l01719"></a>01719     detector_data[0].<a class="code" href="structdetector__struct.html#adbe41a01886ae097dac0fa908551d035">rotation_flag</a> = 0;
<a name="l01720"></a>01720     detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = detector_center.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>;
<a name="l01721"></a>01721     detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = detector_center.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>;
<a name="l01722"></a>01722     detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = detector_center.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>;
<a name="l01723"></a>01723 
<a name="l01724"></a>01724     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Source pointing to (0,1,0): detector not rotated, initial location in voxels found faster.\n&quot;</span>);  <span class="comment">// maximizing code efficiency -&gt; the simulation will be faster than for other angles (but not much).&quot;);</span>
<a name="l01725"></a>01725 
<a name="l01726"></a>01726   }
<a name="l01727"></a>01727   <span class="keywordflow">else</span>
<a name="l01728"></a>01728   { <span class="comment">// Rotation needed to set the detector perpendicular to +Y:</span>
<a name="l01729"></a>01729     detector_data[0].<a class="code" href="structdetector__struct.html#adbe41a01886ae097dac0fa908551d035">rotation_flag</a> = 1;
<a name="l01730"></a>01730     <span class="comment">// -- Rotate the detector center to +Y:</span>
<a name="l01731"></a>01731     detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = detector_center.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*detector_data-&gt;<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[0] + detector_center.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[1] + detector_center.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[2];
<a name="l01732"></a>01732     detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = detector_center.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[3] + detector_center.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[4] + detector_center.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[5];
<a name="l01733"></a>01733     detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = detector_center.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[6] + detector_center.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[7] + detector_center.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[8];
<a name="l01734"></a>01734 
<a name="l01735"></a>01735     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Rotations from the input direction to +Y [deg]: rotZ = %f , rotX = %f\n&quot;</span>, rotZ*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>, rotX*RAD2DEG);
<a name="l01736"></a>01736     
<a name="l01737"></a>01737   }
<a name="l01738"></a>01738   <span class="comment">// -- Set the lower corner (minimum) coordinates at the normalized orientation: +Y. The detector has thickness 0.</span>
<a name="l01739"></a>01739   detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> - 0.5*detector_data[0].<a class="code" href="structdetector__struct.html#a1f67c512205953e75552865ae7a2490a">width_X</a>;
<a name="l01740"></a>01740 <span class="comment">//  detector_data[0].corner_min_rotated_to_Y.y = detector_data[0].corner_min_rotated_to_Y.y;</span>
<a name="l01741"></a>01741   detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> - 0.5*detector_data[0].<a class="code" href="structdetector__struct.html#af61592c77c6da548fff31215bf956dd8">height_Z</a>;
<a name="l01742"></a>01742   
<a name="l01743"></a>01743   detector_data[0].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = source_data[0].position.x + source_data[0].direction.x * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l01744"></a>01744   detector_data[0].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = source_data[0].position.y + source_data[0].direction.y * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l01745"></a>01745   detector_data[0].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = source_data[0].position.z + source_data[0].direction.z * detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l01746"></a>01746 
<a name="l01747"></a>01747   
<a name="l01748"></a>01748   <span class="comment"></span>
<a name="l01749"></a>01749 <span class="comment">  /////////////////////////////////////////////////////////////////////////////</span>
<a name="l01750"></a>01750 <span class="comment"></span>
<a name="l01751"></a>01751   <span class="comment">// *** Init the fan beam source model:</span>
<a name="l01752"></a>01752 
<a name="l01753"></a>01753   <span class="keywordflow">if</span> (1 == detector_data[0].rotation_flag)
<a name="l01754"></a>01754   {
<a name="l01755"></a>01755     <span class="comment">// Initial beam NOT pointing to +Y: rotation is needed to move the sampled vector from (0,1,0) to the given direction!!</span>
<a name="l01756"></a>01756     rotX = 0.5*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a> - acos(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);         <span class="comment">// ! Rotation about X: acos(wsrc)==theta, theta=90 for alpha=0, ie, +Y.</span>
<a name="l01757"></a>01757     rotZ = atan2(source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, source_data[0].direction.x) - 0.5*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// ! Rotation about Z:  initial phi = 90 (+Y).  [ATAN2(v,u) = TAN(v/u), with the angle in the correct quadrant.</span>
<a name="l01758"></a>01758     cos_rX = cos(rotX);
<a name="l01759"></a>01759     cos_rZ = cos(rotZ);
<a name="l01760"></a>01760     sin_rX = sin(rotX);
<a name="l01761"></a>01761     sin_rZ = sin(rotZ);
<a name="l01762"></a>01762     <span class="comment">// --Rotation around X (alpha) and then around Z (phi): Rz*Rx (oposite of detector rotation)</span>
<a name="l01763"></a>01763     source_data[0].rot_fan[0] =  cos_rZ;
<a name="l01764"></a>01764     source_data[0].rot_fan[1] = -cos_rX*sin_rZ;
<a name="l01765"></a>01765     source_data[0].rot_fan[2] =  sin_rX*sin_rZ;
<a name="l01766"></a>01766     source_data[0].rot_fan[3] =  sin_rZ;
<a name="l01767"></a>01767     source_data[0].rot_fan[4] =  cos_rX*cos_rZ;
<a name="l01768"></a>01768     source_data[0].rot_fan[5] = -sin_rX*cos_rZ;
<a name="l01769"></a>01769     source_data[0].rot_fan[6] =  0.0f;
<a name="l01770"></a>01770     source_data[0].rot_fan[7] =  sin_rX;
<a name="l01771"></a>01771     source_data[0].rot_fan[8] =  cos_rX;
<a name="l01772"></a>01772     
<a name="l01773"></a>01773     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Rotations from +Y to the input direction for the fan beam source model [deg]: rotZ = %f , rotX = %f\n&quot;</span>, rotZ*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>, rotX*RAD2DEG);
<a name="l01774"></a>01774   }      
<a name="l01775"></a>01775 
<a name="l01776"></a>01776 <span class="comment"></span>
<a name="l01777"></a>01777 <span class="comment">  /////////////////////////////////////////////////////////////////////////////</span>
<a name="l01778"></a>01778 <span class="comment"></span>
<a name="l01779"></a>01779 
<a name="l01780"></a>01780   <span class="comment">// *** Allocate array for the 4 detected images (non-scattered, Compton, Rayleigh, multiple-scatter):</span>
<a name="l01781"></a>01781   <span class="keywordtype">int</span> pixels_per_image = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a> * detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>;
<a name="l01782"></a>01782   *image_bytes = 4 * pixels_per_image * <span class="keyword">sizeof</span>(<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> int);
<a name="l01783"></a>01783   (*image_ptr) = (<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>*) malloc(*image_bytes);
<a name="l01784"></a>01784   <span class="keywordflow">if</span> (*image_ptr==NULL)
<a name="l01785"></a>01785   {
<a name="l01786"></a>01786     printf(<span class="stringliteral">&quot;\n\n   !!malloc ERROR!! Not enough memory to allocate %d pixels for the 4 scatter images (%f Mbytes)!!\n\n&quot;</span>, pixels_per_image, (*image_bytes)/(1024.f*1024.f));
<a name="l01787"></a>01787     exit(-2);
<a name="l01788"></a>01788   }
<a name="l01789"></a>01789   <span class="keywordflow">else</span>
<a name="l01790"></a>01790   {
<a name="l01791"></a>01791     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Array for 4 scatter images correctly allocated (%d pixels, %f Mbytes)\n&quot;</span>, pixels_per_image, (*image_bytes)/(1024.f*1024.f));
<a name="l01792"></a>01792   }
<a name="l01793"></a>01793 
<a name="l01794"></a>01794 <span class="preprocessor">#ifndef USING_CUDA</span>
<a name="l01795"></a>01795 <span class="preprocessor"></span>  <span class="comment">// *** Initialize the images to 0 in the CPU. The CUDA code will init it to 0 in the GPU global memory later, using kernel &quot;init_image_array_GPU&quot;.</span>
<a name="l01796"></a>01796   memset(*image_ptr, 0, (*image_bytes));     <span class="comment">// Init memory space to 0.   </span>
<a name="l01797"></a>01797 <span class="preprocessor">#endif</span>
<a name="l01798"></a>01798 <span class="preprocessor"></span>
<a name="l01799"></a>01799 
<a name="l01800"></a>01800   <span class="comment">// *** Allocate dose and dose^2 array if tally active:</span>
<a name="l01801"></a>01801   <span class="keywordtype">int</span> num_voxels_ROI = ((int)(*dose_ROI_x_max - *dose_ROI_x_min + 1)) * ((<span class="keywordtype">int</span>)(*dose_ROI_y_max - *dose_ROI_y_min + 1)) * ((int)(*dose_ROI_z_max - *dose_ROI_z_min + 1));
<a name="l01802"></a>01802   <span class="keywordflow">if</span> ((*dose_ROI_x_max)&gt;-1)
<a name="l01803"></a>01803   {    
<a name="l01804"></a>01804     *voxels_Edep_bytes = num_voxels_ROI * <span class="keyword">sizeof</span>(<a class="code" href="MC-GPU__v1_83_8h.html#a197f1fccb1076eb0f1555b5571130b2b">ulonglong2</a>);
<a name="l01805"></a>01805     (*voxels_Edep_ptr) = (<a class="code" href="structulonglong2.html">ulonglong2</a>*) malloc(*voxels_Edep_bytes);
<a name="l01806"></a>01806     <span class="keywordflow">if</span> (*voxels_Edep_ptr==NULL)
<a name="l01807"></a>01807     {
<a name="l01808"></a>01808       printf(<span class="stringliteral">&quot;\n\n   !!malloc ERROR!! Not enough memory to allocate %d voxels for the deposited dose (and uncertainty) array (%f Mbytes)!!\n\n&quot;</span>, num_voxels_ROI, (*voxels_Edep_bytes)/(1024.f*1024.f));
<a name="l01809"></a>01809       exit(-2);
<a name="l01810"></a>01810     }
<a name="l01811"></a>01811     <span class="keywordflow">else</span>
<a name="l01812"></a>01812     {
<a name="l01813"></a>01813       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Array for the deposited dose ROI (and uncertainty) correctly allocated (%d voxels, %f Mbytes)\n&quot;</span>, num_voxels_ROI, (*voxels_Edep_bytes)/(1024.f*1024.f));
<a name="l01814"></a>01814     }
<a name="l01815"></a>01815   }
<a name="l01816"></a>01816   <span class="keywordflow">else</span>
<a name="l01817"></a>01817   {
<a name="l01818"></a>01818     (*voxels_Edep_bytes) = 0;
<a name="l01819"></a>01819   }
<a name="l01820"></a>01820   
<a name="l01821"></a>01821   <span class="comment">// *** Initialize the voxel dose to 0 in the CPU. Not necessary for the CUDA code if dose matrix init. in the GPU global memory using a GPU kernel, but needed if using cudaMemcpy.  </span>
<a name="l01822"></a>01822   <span class="keywordflow">if</span> ((*dose_ROI_x_max)&gt;-1)
<a name="l01823"></a>01823   {    
<a name="l01824"></a>01824     memset(*voxels_Edep_ptr, 0, (*voxels_Edep_bytes));     <span class="comment">// Init memory space to 0.</span>
<a name="l01825"></a>01825   }
<a name="l01826"></a>01826 
<a name="l01827"></a>01827   <span class="keywordflow">return</span>;
<a name="l01828"></a>01828 }
<a name="l01829"></a>01829 
<a name="l01830"></a>01830 
<a name="l01831"></a>01831 <span class="comment"></span>
<a name="l01832"></a>01832 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01833"></a>01833 <span class="comment">//! Extract a file name from an input text line, trimming the initial blanks,</span>
<a name="l01834"></a>01834 <span class="comment">//! trailing comment (#) and stopping at the first blank (the file name should</span>
<a name="l01835"></a>01835 <span class="comment">//! not contain blanks).</span>
<a name="l01836"></a>01836 <span class="comment">//!</span>
<a name="l01837"></a>01837 <span class="comment">//!       @param[in] input_line   Input sentence with blanks and a trailing comment</span>
<a name="l01838"></a>01838 <span class="comment">//!       @param[out] file_name   Trimmed file name</span>
<a name="l01839"></a>01839 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01840"></a><a class="code" href="MC-GPU__v1_83_8h.html#a96dba25b308814b25fbae71e27617f12">01840</a> <span class="comment"></span><span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a96dba25b308814b25fbae71e27617f12" title="Extract a file name from an input text line, trimming the initial blanks, trailing comment (#) and st...">trim_name</a>(<span class="keywordtype">char</span>* input_line, <span class="keywordtype">char</span>* file_name)
<a name="l01841"></a>01841 {
<a name="l01842"></a>01842   <span class="keywordtype">int</span> a=0, b=0;
<a name="l01843"></a>01843   
<a name="l01844"></a>01844   <span class="comment">// Discard initial blanks:</span>
<a name="l01845"></a>01845   <span class="keywordflow">while</span>(<span class="charliteral">&#39; &#39;</span>==input_line[a])
<a name="l01846"></a>01846   {
<a name="l01847"></a>01847     a++;
<a name="l01848"></a>01848   }
<a name="l01849"></a>01849 
<a name="l01850"></a>01850   <span class="comment">// Read file name until a blank or a comment symbol (#) is found:</span>
<a name="l01851"></a>01851   <span class="keywordflow">while</span> ((<span class="charliteral">&#39; &#39;</span>!=input_line[a])&amp;&amp;(<span class="charliteral">&#39;#&#39;</span>!=input_line[a]))
<a name="l01852"></a>01852   {
<a name="l01853"></a>01853     file_name[b] = input_line[a];
<a name="l01854"></a>01854     b++;
<a name="l01855"></a>01855     a++;
<a name="l01856"></a>01856   }
<a name="l01857"></a>01857   
<a name="l01858"></a>01858   file_name[b] = <span class="charliteral">&#39;\0&#39;</span>;    <span class="comment">// Terminate output string</span>
<a name="l01859"></a>01859 }
<a name="l01860"></a>01860 <span class="comment"></span>
<a name="l01861"></a>01861 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01862"></a>01862 <span class="comment">//! Read a line of text and trim initial blancks and trailing comments (#).</span>
<a name="l01863"></a>01863 <span class="comment">//!</span>
<a name="l01864"></a>01864 <span class="comment">//!       @param[in] num   Characters to read</span>
<a name="l01865"></a>01865 <span class="comment">//!       @param[in] file_ptr   Pointer to the input file stream</span>
<a name="l01866"></a>01866 <span class="comment">//!       @param[out] trimmed_line   Trimmed line from input file, skipping empty lines and comments</span>
<a name="l01867"></a>01867 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01868"></a><a class="code" href="MC-GPU__v1_83_8h.html#aea168bca1466813e03b649931a4f670f">01868</a> <span class="comment"></span><span class="keywordtype">char</span>* <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(<span class="keywordtype">char</span>* trimmed_line, <span class="keywordtype">int</span> num, FILE* file_ptr)
<a name="l01869"></a>01869 {
<a name="l01870"></a>01870   <span class="keywordtype">char</span>  new_line[250];
<a name="l01871"></a>01871   <span class="keywordtype">char</span> *new_line_ptr = NULL;
<a name="l01872"></a>01872   <span class="keywordtype">int</span> a=0, b=0;
<a name="l01873"></a>01873   trimmed_line[0] = <span class="charliteral">&#39;\0&#39;</span>;   <span class="comment">//  Init with a mark that means no file input</span>
<a name="l01874"></a>01874   
<a name="l01875"></a>01875   <span class="keywordflow">do</span>
<a name="l01876"></a>01876   {
<a name="l01877"></a>01877     a=0; b=0;
<a name="l01878"></a>01878     new_line_ptr = fgets(new_line, num, file_ptr);   <span class="comment">// Read new line</span>
<a name="l01879"></a>01879     <span class="keywordflow">if</span> (new_line_ptr != NULL)
<a name="l01880"></a>01880     {
<a name="l01881"></a>01881       <span class="comment">// Discard initial blanks:</span>
<a name="l01882"></a>01882       <span class="keywordflow">while</span>(<span class="charliteral">&#39; &#39;</span>==new_line[a])
<a name="l01883"></a>01883       {
<a name="l01884"></a>01884         a++;
<a name="l01885"></a>01885       }
<a name="l01886"></a>01886       <span class="comment">// Read file until a comment symbol (#) or end-of-line are found:</span>
<a name="l01887"></a>01887       <span class="keywordflow">while</span> ((<span class="charliteral">&#39;\n&#39;</span>!=new_line[a])&amp;&amp;(<span class="charliteral">&#39;#&#39;</span>!=new_line[a]))
<a name="l01888"></a>01888       {
<a name="l01889"></a>01889         trimmed_line[b] = new_line[a];
<a name="l01890"></a>01890         b++;
<a name="l01891"></a>01891         a++;
<a name="l01892"></a>01892       }
<a name="l01893"></a>01893     }
<a name="l01894"></a>01894   } <span class="keywordflow">while</span>(new_line_ptr!=NULL &amp;&amp;  <span class="charliteral">&#39;\0&#39;</span>==trimmed_line[0]);   <span class="comment">// Keep reading lines until end-of-file or a line that is not empty or only comment is found</span>
<a name="l01895"></a>01895   
<a name="l01896"></a>01896   trimmed_line[b] = <span class="charliteral">&#39;\0&#39;</span>;    <span class="comment">// Terminate output string</span>
<a name="l01897"></a>01897   <span class="keywordflow">return</span> new_line_ptr;
<a name="l01898"></a>01898 }
<a name="l01899"></a>01899 
<a name="l01900"></a>01900 
<a name="l01901"></a>01901 <span class="comment"></span>
<a name="l01902"></a>01902 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01903"></a>01903 <span class="comment">//! Read the voxel data and allocate the material and density matrix.</span>
<a name="l01904"></a>01904 <span class="comment">//! Also find and report the maximum density defined in the geometry.</span>
<a name="l01905"></a>01905 <span class="comment">//!</span>
<a name="l01906"></a>01906 <span class="comment"></span><span class="comment">// -- Sample voxel geometry file:</span>
<a name="l01907"></a>01907 <span class="comment">//</span>
<a name="l01908"></a>01908 <span class="comment">//   #  (comment lines...)</span>
<a name="l01909"></a>01909 <span class="comment">//   #</span>
<a name="l01910"></a>01910 <span class="comment">//   #   Voxel order: X runs first, then Y, then Z.</span>
<a name="l01911"></a>01911 <span class="comment">//   #</span>
<a name="l01912"></a>01912 <span class="comment">//   [SECTION VOXELS HEADER v.2008-04-13]</span>
<a name="l01913"></a>01913 <span class="comment">//   411  190  113      No. OF VOXELS IN X,Y,Z</span>
<a name="l01914"></a>01914 <span class="comment">//   5.000e-02  5.000e-02  5.000e-02    VOXEL SIZE (cm) ALONG X,Y,Z</span>
<a name="l01915"></a>01915 <span class="comment">//   1                  COLUMN NUMBER WHERE MATERIAL ID IS LOCATED</span>
<a name="l01916"></a>01916 <span class="comment">//   2                  COLUMN NUMBER WHERE THE MASS DENSITY IS LOCATED</span>
<a name="l01917"></a>01917 <span class="comment">//   1                  BLANK LINES AT END OF X,Y-CYCLES (1=YES,0=NO)</span>
<a name="l01918"></a>01918 <span class="comment">//   [END OF VXH SECTION]</span>
<a name="l01919"></a>01919 <span class="comment">//   1 0.00120479</span>
<a name="l01920"></a>01920 <span class="comment">//   1 0.00120479</span>
<a name="l01921"></a>01921 <span class="comment">//   ...</span>
<a name="l01922"></a>01922 <span class="comment">//</span><span class="comment"></span>
<a name="l01923"></a>01923 <span class="comment">//!       @param[in] file_name_voxels  Name of the voxelized geometry file.</span>
<a name="l01924"></a>01924 <span class="comment">//!       @param[out] density_max  Array with the maximum density for each material in the voxels.</span>
<a name="l01925"></a>01925 <span class="comment">//!       @param[out] voxel_data   Pointer to a structure containing the voxel number and size.</span>
<a name="l01926"></a>01926 <span class="comment">//!       @param[out] voxel_mat_dens_ptr   Pointer to the vector with the voxel materials and densities.</span>
<a name="l01927"></a>01927 <span class="comment">//!       @param[in] dose_ROI_x/y/z_max   Size of the dose ROI: can not be larger than the total number of voxels in the geometry.</span>
<a name="l01928"></a>01928 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l01929"></a><a class="code" href="MC-GPU__v1_83_8h.html#aed61d205baa31cea69768bbd68c062c6">01929</a> <span class="comment"></span><span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#aed61d205baa31cea69768bbd68c062c6" title="Read the voxel data and allocate the material and density matrix.">load_voxels</a>(<span class="keywordtype">int</span> myID, <span class="keywordtype">char</span>* file_name_voxels, <span class="keywordtype">float</span>* density_max, <span class="keyword">struct</span> <a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a>* voxel_data, <a class="code" href="structfloat2.html">float2</a>** voxel_mat_dens_ptr, <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span>* voxel_mat_dens_bytes, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_x_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_y_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_z_max)
<a name="l01930"></a>01930 {
<a name="l01931"></a>01931   <span class="keywordtype">char</span> new_line[250];
<a name="l01932"></a>01932   <span class="keywordtype">char</span> *new_line_ptr = NULL;  
<a name="l01933"></a>01933       
<a name="l01934"></a>01934   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> <span class="keywordflow">if</span> (strstr(file_name_voxels,<span class="stringliteral">&quot;.zip&quot;</span>)!=NULL)
<a name="l01935"></a>01935     printf(<span class="stringliteral">&quot;\n\n    -- WARNING load_voxels! The input voxel file name has the extension \&#39;.zip\&#39;. Only \&#39;.gz\&#39; compression is allowed!!\n\n&quot;</span>);     <span class="comment">// !!zlib!!</span>
<a name="l01936"></a>01936     
<a name="l01937"></a>01937   gzFile file_ptr = gzopen(file_name_voxels, <span class="stringliteral">&quot;rb&quot;</span>);  <span class="comment">// Open the file with zlib: the file can be compressed with gzip or uncompressed.   !!zlib!!  </span>
<a name="l01938"></a>01938   
<a name="l01939"></a>01939   <span class="keywordflow">if</span> (file_ptr==NULL)
<a name="l01940"></a>01940   {
<a name="l01941"></a>01941     printf(<span class="stringliteral">&quot;\n\n   !! fopen ERROR load_voxels!! File %s does not exist!!\n&quot;</span>, file_name_voxels);
<a name="l01942"></a>01942     exit(-2);
<a name="l01943"></a>01943   }
<a name="l01944"></a>01944   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> 
<a name="l01945"></a>01945   {
<a name="l01946"></a>01946     printf(<span class="stringliteral">&quot;\n    -- Reading voxel file \&#39;%s\&#39;:\n&quot;</span>,file_name_voxels);
<a name="l01947"></a>01947     <span class="keywordflow">if</span> (strstr(file_name_voxels,<span class="stringliteral">&quot;.gz&quot;</span>)==NULL)
<a name="l01948"></a>01948       printf(<span class="stringliteral">&quot;         (note that MC-GPU can also read voxel and material files compressed with gzip)\n&quot;</span>);     <span class="comment">// !!zlib!!  </span>
<a name="l01949"></a>01949     fflush(stdout);
<a name="l01950"></a>01950   }
<a name="l01951"></a>01951   <span class="keywordflow">do</span>
<a name="l01952"></a>01952   {    
<a name="l01953"></a>01953     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l01954"></a>01954     
<a name="l01955"></a>01955     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01956"></a>01956     {
<a name="l01957"></a>01957       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!Reading ERROR load_voxels!! File is not readable or does not contain the string \&#39;[SECTION VOXELS HEADER\&#39;!!\n&quot;</span>);
<a name="l01958"></a>01958       exit(-2);
<a name="l01959"></a>01959     }
<a name="l01960"></a>01960   }
<a name="l01961"></a>01961   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;[SECTION VOXELS&quot;</span>)==NULL);   <span class="comment">// Skip comments and empty lines until the header begins</span>
<a name="l01962"></a>01962 
<a name="l01963"></a>01963   <a class="code" href="structfloat3.html">float3</a> voxel_size;
<a name="l01964"></a>01964   new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!   // Read full line (max. 250 characters).</span>
<a name="l01965"></a>01965   sscanf(new_line, <span class="stringliteral">&quot;%d %d %d&quot;</span>,&amp;voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>, &amp;voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>, &amp;voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>);
<a name="l01966"></a>01966   new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l01967"></a>01967   sscanf(new_line, <span class="stringliteral">&quot;%f %f %f&quot;</span>, &amp;voxel_size.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>, &amp;voxel_size.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, &amp;voxel_size.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l01968"></a>01968   <span class="keywordflow">do</span>
<a name="l01969"></a>01969   {
<a name="l01970"></a>01970     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l01971"></a>01971     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l01972"></a>01972     {
<a name="l01973"></a>01973       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n\n   !!Reading ERROR load_voxels!! File is not readable or does not contain the string \&#39;[END OF VXH SECTION]\&#39;!!\n&quot;</span>);
<a name="l01974"></a>01974       exit(-2);
<a name="l01975"></a>01975     }
<a name="l01976"></a>01976   }
<a name="l01977"></a>01977   <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;[END OF VXH SECTION&quot;</span>)==NULL);   <span class="comment">// Skip rest of the header</span>
<a name="l01978"></a>01978 
<a name="l01979"></a>01979   <span class="comment">// -- Store the size of the voxel bounding box (used in the source function):</span>
<a name="l01980"></a>01980   voxel_data-&gt;<a class="code" href="structvoxel__struct.html#abdd12181cdff77da8b880eedf1d81e33">size_bbox</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a> * voxel_size.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>;
<a name="l01981"></a>01981   voxel_data-&gt;<a class="code" href="structvoxel__struct.html#abdd12181cdff77da8b880eedf1d81e33">size_bbox</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a> * voxel_size.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>;
<a name="l01982"></a>01982   voxel_data-&gt;<a class="code" href="structvoxel__struct.html#abdd12181cdff77da8b880eedf1d81e33">size_bbox</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a> * voxel_size.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>;
<a name="l01983"></a>01983   
<a name="l01984"></a>01984   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Number of voxels in the input geometry file: %d x %d x %d =  %d\n&quot;</span>, voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>, voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>, voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>, (voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>));
<a name="l01985"></a>01985   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Size of the input voxels: %f x %f x %f cm  (voxel volume=%f cm^3)\n&quot;</span>, voxel_size.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>, voxel_size.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, voxel_size.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>, voxel_size.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*voxel_size.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*voxel_size.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l01986"></a>01986   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Voxel bounding box size: %f x %f x %f cm\n&quot;</span>, voxel_data-&gt;<a class="code" href="structvoxel__struct.html#abdd12181cdff77da8b880eedf1d81e33">size_bbox</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>, voxel_data-&gt;<a class="code" href="structvoxel__struct.html#abdd12181cdff77da8b880eedf1d81e33">size_bbox</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>,  voxel_data-&gt;<a class="code" href="structvoxel__struct.html#abdd12181cdff77da8b880eedf1d81e33">size_bbox</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l01987"></a>01987   <span class="comment">// printf(&quot;       The geometry must be given in two columns, with the voxel density in the second column.\n&quot;);</span>
<a name="l01988"></a>01988   <span class="comment">// printf(&quot;       The  X,Y-cycles may, or may not, be separated by blank lines.\n&quot;);</span>
<a name="l01989"></a>01989 
<a name="l01990"></a>01990   <span class="comment">// -- Make sure the input number of voxels in the vox file is compatible with the input dose ROI (ROI assumes first voxel is index 0):</span>
<a name="l01991"></a>01991   <span class="keywordflow">if</span> ( (*dose_ROI_x_max+1)&gt;(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>) || (*dose_ROI_y_max+1)&gt;(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>) || (*dose_ROI_z_max+1)&gt;(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>) )
<a name="l01992"></a>01992   {
<a name="l01993"></a>01993     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n       The input region of interest for the dose deposition is larger than the size of the voxelized geometry:\n&quot;</span>);
<a name="l01994"></a>01994     *dose_ROI_x_max = <a class="code" href="MC-GPU__v1_83_8h.html#a9fa6cc963655f85859604988fa68a9e2">min_value</a>(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>-1, *dose_ROI_x_max);
<a name="l01995"></a>01995     *dose_ROI_y_max = <a class="code" href="MC-GPU__v1_83_8h.html#a9fa6cc963655f85859604988fa68a9e2">min_value</a>(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>-1, *dose_ROI_y_max);
<a name="l01996"></a>01996     *dose_ROI_z_max = <a class="code" href="MC-GPU__v1_83_8h.html#a9fa6cc963655f85859604988fa68a9e2">min_value</a>(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>-1, *dose_ROI_z_max);
<a name="l01997"></a>01997     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(  <span class="stringliteral">&quot;       updating the ROI max limits to fit the geometry -&gt; dose_ROI_max=(%d, %d, %d)\n&quot;</span>, *dose_ROI_x_max+1, *dose_ROI_y_max+1, *dose_ROI_z_max+1);         <span class="comment">// Allowing the input of an ROI larger than the voxel volume: in this case some of the allocated memory will be wasted but the program will run ok.</span>
<a name="l01998"></a>01998   }
<a name="l01999"></a>01999   
<a name="l02000"></a>02000   <span class="keywordflow">if</span> ( (*dose_ROI_x_max+1)==(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>) &amp;&amp; (*dose_ROI_y_max+1)==(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>) &amp;&amp; (*dose_ROI_z_max+1)==(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>) )
<a name="l02001"></a>02001     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       The voxel dose tally ROI covers the entire voxelized phantom: the dose to every voxel will be tallied.\n&quot;</span>);
<a name="l02002"></a>02002   <span class="keywordflow">else</span>
<a name="l02003"></a>02003     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       The voxel dose tally ROI covers only a fraction of the voxelized phantom: the dose to voxels outside the ROI will not be tallied.\n&quot;</span>);
<a name="l02004"></a>02004  
<a name="l02005"></a>02005   <span class="comment">// -- Store the inverse of the pixel sides (in cm) to speed up the particle location in voxels.</span>
<a name="l02006"></a>02006   voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = 1.0f/(voxel_size.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>);
<a name="l02007"></a>02007   voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = 1.0f/(voxel_size.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>);
<a name="l02008"></a>02008   voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = 1.0f/(voxel_size.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l02009"></a>02009   
<a name="l02010"></a>02010   <span class="comment">// -- Allocate the voxel matrix and store array size:</span>
<a name="l02011"></a>02011   *voxel_mat_dens_bytes = <span class="keyword">sizeof</span>(<a class="code" href="MC-GPU__v1_83_8h.html#ac5ce49099e54cbb6e687df38a6f3fe96">float2</a>)*(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>)*(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>)*(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>);
<a name="l02012"></a>02012   *voxel_mat_dens_ptr    = (<a class="code" href="structfloat2.html">float2</a>*) malloc(*voxel_mat_dens_bytes);
<a name="l02013"></a>02013   <span class="keywordflow">if</span> (*voxel_mat_dens_ptr==NULL)
<a name="l02014"></a>02014   {
<a name="l02015"></a>02015     printf(<span class="stringliteral">&quot;\n\n   !!malloc ERROR load_voxels!! Not enough memory to allocate %d voxels (%f Mbytes)!!\n\n&quot;</span>, (voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>), (*voxel_mat_dens_bytes)/(1024.f*1024.f));
<a name="l02016"></a>02016     exit(-2);
<a name="l02017"></a>02017   }
<a name="l02018"></a>02018   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n    -- Initializing the voxel material and density vector (%f Mbytes)...\n&quot;</span>, (*voxel_mat_dens_bytes)/(1024.f*1024.f));
<a name="l02019"></a>02019   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> fflush(stdout);
<a name="l02020"></a>02020   
<a name="l02021"></a>02021   <span class="comment">// -- Read the voxel densities:</span>
<a name="l02022"></a>02022   <span class="comment">//   MASTER_THREAD printf(&quot;       Reading the voxel densities... &quot;);</span>
<a name="l02023"></a>02023   <span class="keywordtype">int</span> i, j, k, read_lines=0, dummy_material, read_items = -99;
<a name="l02024"></a>02024   <span class="keywordtype">float</span> dummy_density;
<a name="l02025"></a>02025   <a class="code" href="structfloat2.html">float2</a> *voxels_ptr = *voxel_mat_dens_ptr;
<a name="l02026"></a>02026 
<a name="l02027"></a>02027   <span class="keywordflow">for</span> (k=0; k&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; k++)
<a name="l02028"></a>02028     density_max[k] = -999.0f;   <span class="comment">// Init array with an impossible low density value</span>
<a name="l02029"></a>02029   
<a name="l02030"></a>02030 
<a name="l02031"></a>02031   <span class="keywordflow">for</span>(k=0; k&lt;(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5cd5a3c388fa28814e3496ef07c39360">z</a>); k++)
<a name="l02032"></a>02032   {
<a name="l02033"></a>02033     <span class="keywordflow">for</span>(j=0; j&lt;(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>); j++)
<a name="l02034"></a>02034     {
<a name="l02035"></a>02035       <span class="keywordflow">for</span>(i=0; i&lt;(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>); i++)
<a name="l02036"></a>02036       {
<a name="l02037"></a>02037         
<a name="l02038"></a>02038         <span class="keywordflow">do</span>
<a name="l02039"></a>02039         {
<a name="l02040"></a>02040           new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02041"></a>02041         } 
<a name="l02042"></a>02042         <span class="keywordflow">while</span> ((<span class="charliteral">&#39;\n&#39;</span>==new_line[0])||(<span class="charliteral">&#39;\n&#39;</span>==new_line[1])||(<span class="charliteral">&#39;#&#39;</span>==new_line[0])||(<span class="charliteral">&#39;#&#39;</span>==new_line[1]));   <span class="comment">// Skip empty lines and comments.</span>
<a name="l02043"></a>02043         read_items = sscanf(new_line, <span class="stringliteral">&quot;%d %f&quot;</span>, &amp;dummy_material, &amp;dummy_density);    <span class="comment">// Read the next 2 numbers</span>
<a name="l02044"></a>02044   
<a name="l02045"></a>02045         <span class="keywordflow">if</span> (read_items!=2)
<a name="l02046"></a>02046           printf(<span class="stringliteral">&quot;\n   !!WARNING load_voxels!! Expecting to read 2 items (material and density). read_items=%d, read_lines=%d \n&quot;</span>, read_items, read_lines);
<a name="l02047"></a>02047   
<a name="l02048"></a>02048         <span class="keywordflow">if</span> (dummy_material&gt;MAX_MATERIALS)
<a name="l02049"></a>02049         {
<a name="l02050"></a>02050           printf(<span class="stringliteral">&quot;\n\n   !!ERROR load_voxels!! Voxel material number too high!! #mat=%d, MAX_MATERIALS=%d, voxel number=%d\n\n&quot;</span>, dummy_material, MAX_MATERIALS, read_lines+1);
<a name="l02051"></a>02051           exit(-2);
<a name="l02052"></a>02052         }
<a name="l02053"></a>02053         <span class="keywordflow">if</span> (dummy_material&lt;1)
<a name="l02054"></a>02054         {
<a name="l02055"></a>02055           printf(<span class="stringliteral">&quot;\n\n   !!ERROR load_voxels!! Voxel material number can not be zero or negative!! #mat=%d, voxel number=%dd\n\n&quot;</span>, dummy_material, read_lines+1);
<a name="l02056"></a>02056           exit(-2);
<a name="l02057"></a>02057         }
<a name="l02058"></a>02058         
<a name="l02059"></a>02059         <span class="keywordflow">if</span> (dummy_density &lt; 1.0e-9f)
<a name="l02060"></a>02060         {
<a name="l02061"></a>02061           printf(<span class="stringliteral">&quot;\n\n   !!ERROR load_voxels!! Voxel density can not be 0 or negative: #mat=%d, density=%f, voxel number=%d\n\n&quot;</span>, dummy_material, dummy_density, read_lines+1);
<a name="l02062"></a>02062           exit(-2);
<a name="l02063"></a>02063         }        
<a name="l02064"></a>02064         
<a name="l02065"></a>02065         <span class="keywordflow">if</span> (dummy_density &gt; density_max[dummy_material-1])
<a name="l02066"></a>02066           density_max[dummy_material-1] = dummy_density;  <span class="comment">// Store maximum density for each material</span>
<a name="l02067"></a>02067 
<a name="l02068"></a>02068         (*voxels_ptr).x = (float)(dummy_material)+0.0001f;  <span class="comment">// Assign material value as float (the integer value will be recovered by truncation)</span>
<a name="l02069"></a>02069         (*voxels_ptr).y = dummy_density;      <span class="comment">// Assign density value</span>
<a name="l02070"></a>02070         voxels_ptr++;                         <span class="comment">// Move to next voxel</span>
<a name="l02071"></a>02071 
<a name="l02072"></a>02072         read_lines++;
<a name="l02073"></a>02073       }
<a name="l02074"></a>02074     }
<a name="l02075"></a>02075   }
<a name="l02076"></a>02076   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       Total number of voxels read: %d\n&quot;</span>,read_lines);
<a name="l02077"></a>02077   gzclose(file_ptr);     <span class="comment">// Close input file    !!zlib!!</span>
<a name="l02078"></a>02078 }
<a name="l02079"></a>02079 
<a name="l02080"></a>02080 <span class="comment"></span>
<a name="l02081"></a>02081 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02082"></a>02082 <span class="comment">//! Read the material input files and set the mean free paths and the &quot;linear_interp&quot; structures.</span>
<a name="l02083"></a>02083 <span class="comment">//! Find the material nominal density. Set the Woodcock trick data.</span>
<a name="l02084"></a>02084 <span class="comment"></span><span class="comment">//</span>
<a name="l02085"></a>02085 <span class="comment">// -- Sample material data file (data obtained from the PENELOPE 2006 database and models):</span>
<a name="l02086"></a>02086 <span class="comment">//</span>
<a name="l02087"></a>02087 <span class="comment">//    [MATERIAL NAME]</span>
<a name="l02088"></a>02088 <span class="comment">//     Water</span>
<a name="l02089"></a>02089 <span class="comment">//    [NOMINAL DENSITY (g/cm^3)]</span>
<a name="l02090"></a>02090 <span class="comment">//     1.000</span>
<a name="l02091"></a>02091 <span class="comment">//    [NUMBER OF DATA VALUES]</span>
<a name="l02092"></a>02092 <span class="comment">//     4096</span>
<a name="l02093"></a>02093 <span class="comment">//    [MEAN FREE PATHS :: Energy (eV) || Rayleigh | Compton | Photoelectric | Pair-production | TOTAL (cm)]</span>
<a name="l02094"></a>02094 <span class="comment">//     1.00000E+03  7.27451E-01  9.43363E+01  2.45451E-04  1.00000E+35  2.45367E-04</span>
<a name="l02095"></a>02095 <span class="comment">//     5.00000E+03  1.80004E+00  8.35996E+00  2.38881E-02  1.00000E+35  2.35089E-02</span>
<a name="l02096"></a>02096 <span class="comment">//     1.00000E+04  4.34941E+00  6.26746E+00  2.02568E-01  1.00000E+35  1.87755E-01</span>
<a name="l02097"></a>02097 <span class="comment">//     ...</span>
<a name="l02098"></a>02098 <span class="comment">//     #[RAYLEIGH INTERACTIONS (RITA sampling  of atomic form factor from EPDL database)]</span>
<a name="l02099"></a>02099 <span class="comment">//     ...</span>
<a name="l02100"></a>02100 <span class="comment">//     #[COMPTON INTERACTIONS (relativistic impulse model with approximated one-electron analytical profiles)]</span>
<a name="l02101"></a>02101 <span class="comment">//     ...</span>
<a name="l02102"></a>02102 <span class="comment">//</span><span class="comment"></span>
<a name="l02103"></a>02103 <span class="comment">//!       @param[in] file_name_materials    Array with the names of the material files.</span>
<a name="l02104"></a>02104 <span class="comment">//!       @param[in] density_max   maximum density in the geometry (needed to set Woodcock trick)</span>
<a name="l02105"></a>02105 <span class="comment">//!       @param[out] density_nominal   Array with the nominal density of the materials read</span>
<a name="l02106"></a>02106 <span class="comment">//!       @param[out] mfp_table_data   Constant values for the linear interpolation</span>
<a name="l02107"></a>02107 <span class="comment">//!       @param[out] mfp_table_a_ptr   First element for the linear interpolation.</span>
<a name="l02108"></a>02108 <span class="comment">//!       @param[out] mfp_table_b_ptr   Second element for the linear interpolation.</span>
<a name="l02109"></a>02109 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02110"></a><a class="code" href="MC-GPU__v1_83_8h.html#ad23b50cb34b4aadcd24d9fec7e47ad24">02110</a> <span class="comment"></span><span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a55f3171a6cdb59fc43e65aa19c676d80" title="Read the material input files and set the mean free paths and the &quot;linear_interp&quot; structures...">load_material</a>(<span class="keywordtype">int</span> myID, <span class="keywordtype">char</span> file_name_materials[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>][250], <span class="keywordtype">float</span>* density_max, <span class="keywordtype">float</span>* density_nominal, <span class="keyword">struct</span> <a class="code" href="structlinear__interp.html" title="Structure with the basic data required by the linear interpolation of the mean free paths: number of ...">linear_interp</a>* mfp_table_data, <a class="code" href="structfloat2.html">float2</a>** mfp_Woodcock_table_ptr, <span class="keywordtype">int</span>* mfp_Woodcock_table_bytes, <a class="code" href="structfloat3.html">float3</a>** mfp_table_a_ptr, <a class="code" href="structfloat3.html">float3</a>** mfp_table_b_ptr, <span class="keywordtype">int</span>* mfp_table_bytes, <span class="keyword">struct</span> <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a> *rayleigh_table_ptr, <span class="keyword">struct</span> <a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a> *compton_table_ptr)
<a name="l02111"></a>02111 {
<a name="l02112"></a>02112   <span class="keywordtype">char</span> new_line[250];
<a name="l02113"></a>02113   <span class="keywordtype">char</span> *new_line_ptr = NULL;
<a name="l02114"></a>02114   <span class="keywordtype">int</span> mat, i, bin, input_num_values = 0, input_rayleigh_values = 0, input_num_shells = 0;
<a name="l02115"></a>02115   <span class="keywordtype">double</span> delta_e=-99999.0;
<a name="l02116"></a>02116 
<a name="l02117"></a>02117   <span class="comment">// -- Init the number of shells to 0 for all materials</span>
<a name="l02118"></a>02118   <span class="keywordflow">for</span> (mat=0; mat&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; mat++)
<a name="l02119"></a>02119     compton_table_ptr-&gt;<a class="code" href="structcompton__struct.html#aef81eb007c90d13552ea867912f0d2c3">noscco</a>[mat] = 0;
<a name="l02120"></a>02120     
<a name="l02121"></a>02121 
<a name="l02122"></a>02122   <span class="comment">// --Read the material data files:</span>
<a name="l02123"></a>02123   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n    -- Reading the material data files (MAX_MATERIALS=%d):\n&quot;</span>, MAX_MATERIALS);
<a name="l02124"></a>02124   <span class="keywordflow">for</span> (mat=0; mat&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; mat++)
<a name="l02125"></a>02125   {
<a name="l02126"></a>02126     <span class="keywordflow">if</span> ((file_name_materials[mat][0]==<span class="charliteral">&#39;\0&#39;</span>) || (file_name_materials[mat][0]==<span class="charliteral">&#39;\n&#39;</span>))  <span class="comment">//  Empty file name</span>
<a name="l02127"></a>02127        <span class="keywordflow">continue</span>;   <span class="comment">// Re-start loop for next material</span>
<a name="l02128"></a>02128 
<a name="l02129"></a>02129     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;         Mat %d: File \&#39;%s\&#39;\n&quot;</span>, mat+1, file_name_materials[mat]);
<a name="l02130"></a>02130 <span class="comment">//     printf(&quot;    -- Reading material file #%d: \&#39;%s\&#39;\n&quot;, mat, file_name_materials[mat]);</span>
<a name="l02131"></a>02131 
<a name="l02132"></a>02132     gzFile file_ptr = gzopen(file_name_materials[mat], <span class="stringliteral">&quot;rb&quot;</span>);    <span class="comment">// !!zlib!!  </span>
<a name="l02133"></a>02133     <span class="keywordflow">if</span> (file_ptr==NULL)
<a name="l02134"></a>02134     {
<a name="l02135"></a>02135       printf(<span class="stringliteral">&quot;\n\n   !!fopen ERROR!! File %d \&#39;%s\&#39; does not exist!!\n&quot;</span>, mat, file_name_materials[mat]);
<a name="l02136"></a>02136       exit(-2);
<a name="l02137"></a>02137     }
<a name="l02138"></a>02138     <span class="keywordflow">do</span>
<a name="l02139"></a>02139     {
<a name="l02140"></a>02140       new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">// Read full line (max. 250 characters).   //  !!zlib!!</span>
<a name="l02141"></a>02141       <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l02142"></a>02142       {
<a name="l02143"></a>02143         printf(<span class="stringliteral">&quot;\n\n   !!Reading ERROR!! File is not readable or does not contain the string \&#39;[NOMINAL DENSITY\&#39;!!\n&quot;</span>);
<a name="l02144"></a>02144         exit(-2);
<a name="l02145"></a>02145       }
<a name="l02146"></a>02146     }
<a name="l02147"></a>02147     <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;[NOMINAL DENSITY&quot;</span>)==NULL);   <span class="comment">// Skip rest of the header</span>
<a name="l02148"></a>02148 
<a name="l02149"></a>02149     <span class="comment">// Read the material nominal density:</span>
<a name="l02150"></a>02150     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02151"></a>02151     sscanf(new_line, <span class="stringliteral">&quot;# %f&quot;</span>, &amp;density_nominal[mat]);
<a name="l02152"></a>02152     
<a name="l02153"></a>02153     <span class="keywordflow">if</span> (density_max[mat]&gt;0)    <span class="comment">//  Material found in the voxels</span>
<a name="l02154"></a>02154     {
<a name="l02155"></a>02155       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                Nominal density = %f g/cm^3; Max density in voxels = %f g/cm^3\n&quot;</span>, density_nominal[mat], density_max[mat]);
<a name="l02156"></a>02156     }
<a name="l02157"></a>02157     <span class="keywordflow">else</span>                       <span class="comment">//  Material NOT found in the voxels</span>
<a name="l02158"></a>02158     {
<a name="l02159"></a>02159       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                This material is not used in any voxel.\n&quot;</span>);
<a name="l02160"></a>02160       
<a name="l02161"></a>02161       <span class="comment">// Do not lose time reading the data for materials not found in the voxels, except for the first one (needed to determine the size of the input data).      </span>
<a name="l02162"></a>02162       <span class="keywordflow">if</span> (0 == mat)
<a name="l02163"></a>02163         density_max[mat] = 0.01f*density_nominal[mat];   <span class="comment">// Assign a small but positive density; this material will not be used anyway.</span>
<a name="l02164"></a>02164       <span class="keywordflow">else</span>
<a name="l02165"></a>02165         <span class="keywordflow">continue</span>;     <span class="comment">//  Move on to next material          </span>
<a name="l02166"></a>02166     }
<a name="l02167"></a>02167       
<a name="l02168"></a>02168 
<a name="l02169"></a>02169     <span class="comment">// --For the first material, set the number of energy values and allocate table arrays:</span>
<a name="l02170"></a>02170     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02171"></a>02171     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02172"></a>02172     sscanf(new_line, <span class="stringliteral">&quot;# %d&quot;</span>, &amp;input_num_values);
<a name="l02173"></a>02173     <span class="keywordflow">if</span> (0==mat)
<a name="l02174"></a>02174     {
<a name="l02175"></a>02175       mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a> = input_num_values;
<a name="l02176"></a>02176       <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                Number of energy values in the mean free path database: %d.\n&quot;</span>, input_num_values);
<a name="l02177"></a>02177 
<a name="l02178"></a>02178       <span class="comment">// Allocate memory for the linear interpolation arrays:</span>
<a name="l02179"></a>02179       *mfp_Woodcock_table_bytes = <span class="keyword">sizeof</span>(<a class="code" href="MC-GPU__v1_83_8h.html#ac5ce49099e54cbb6e687df38a6f3fe96">float2</a>)*input_num_values;
<a name="l02180"></a>02180       *mfp_Woodcock_table_ptr   = (<a class="code" href="structfloat2.html">float2</a>*) malloc(*mfp_Woodcock_table_bytes);  <span class="comment">// Allocate space for the 2 parameter table</span>
<a name="l02181"></a>02181       *mfp_table_bytes = <span class="keyword">sizeof</span>(<a class="code" href="MC-GPU__v1_83_8h.html#a79779bd606ab5af52c38e59729245f27">float3</a>)*input_num_values*MAX_MATERIALS;
<a name="l02182"></a>02182       *mfp_table_a_ptr = (<a class="code" href="structfloat3.html">float3</a>*) malloc(*mfp_table_bytes);  <span class="comment">// Allocate space for the 4 MFP tables</span>
<a name="l02183"></a>02183       *mfp_table_b_ptr = (<a class="code" href="structfloat3.html">float3</a>*) malloc(*mfp_table_bytes);
<a name="l02184"></a>02184       *mfp_table_bytes = <span class="keyword">sizeof</span>(<a class="code" href="MC-GPU__v1_83_8h.html#a79779bd606ab5af52c38e59729245f27">float3</a>)*input_num_values*MAX_MATERIALS;
<a name="l02185"></a>02185 
<a name="l02186"></a>02186       <span class="keywordflow">if</span> (input_num_values&gt;<a class="code" href="MC-GPU__v1_83_8h.html#a6ca3eda8d58b2d848a9d83c17c89a39c">MAX_ENERGYBINS_RAYLEIGH</a>)
<a name="l02187"></a>02187       {
<a name="l02188"></a>02188         printf(<span class="stringliteral">&quot;\n\n   !!load_material ERROR!! Too many energy bins (Input bins=%d): increase parameter MAX_ENERGYBINS_RAYLEIGH=%d!!\n\n&quot;</span>, input_num_values, <a class="code" href="MC-GPU__v1_83_8h.html#a6ca3eda8d58b2d848a9d83c17c89a39c">MAX_ENERGYBINS_RAYLEIGH</a>);
<a name="l02189"></a>02189         exit(-2);
<a name="l02190"></a>02190       }
<a name="l02191"></a>02191       
<a name="l02192"></a>02192       <span class="keywordflow">if</span> ((NULL==*mfp_Woodcock_table_ptr)||(NULL==*mfp_table_a_ptr)||(NULL==*mfp_table_b_ptr))
<a name="l02193"></a>02193       {
<a name="l02194"></a>02194         printf(<span class="stringliteral">&quot;\n\n   !!malloc ERROR!! Not enough memory to allocate the linear interpolation data: %d bytes!!\n\n&quot;</span>, (*mfp_Woodcock_table_bytes+2*(*mfp_table_bytes)));
<a name="l02195"></a>02195         exit(-2);
<a name="l02196"></a>02196       }
<a name="l02197"></a>02197       <span class="keywordflow">else</span>
<a name="l02198"></a>02198       {
<a name="l02199"></a>02199         <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                Linear interpolation data correctly allocated (%f Mbytes)\n&quot;</span>, (*mfp_Woodcock_table_bytes+2*(*mfp_table_bytes))/(1024.f*1024.f));
<a name="l02200"></a>02200       }
<a name="l02201"></a>02201       <span class="keywordflow">for</span> (i=0; i&lt;input_num_values; i++)
<a name="l02202"></a>02202       {
<a name="l02203"></a>02203         (*mfp_Woodcock_table_ptr)[i].x = 99999999.99f;    <span class="comment">// Init this array with a huge MFP, the minimum values are calculated below</span>
<a name="l02204"></a>02204       }
<a name="l02205"></a>02205     }
<a name="l02206"></a>02206     <span class="keywordflow">else</span>   <span class="comment">// Materials after first</span>
<a name="l02207"></a>02207     {
<a name="l02208"></a>02208       <span class="keywordflow">if</span> (input_num_values != mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a>)
<a name="l02209"></a>02209       {
<a name="l02210"></a>02210         printf(<span class="stringliteral">&quot;\n\n   !!load_material ERROR!! Incorrect number of energy values given in material \&#39;%s\&#39;: input=%d, expected=%d\n&quot;</span>,file_name_materials[mat], input_num_values, mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a>);
<a name="l02211"></a>02211         exit(-2);
<a name="l02212"></a>02212       }
<a name="l02213"></a>02213     }
<a name="l02214"></a>02214 
<a name="l02215"></a>02215     <span class="comment">// -- Read the mean free paths (and Rayleigh cumulative prob):</span>
<a name="l02216"></a>02216     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02217"></a>02217     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02218"></a>02218     <span class="keywordtype">double</span> d_energy, d_rayleigh, d_compton, d_photelectric, d_total_mfp, d_pmax, e_last=-1.0;
<a name="l02219"></a>02219     
<a name="l02220"></a>02220     <span class="keywordflow">for</span> (i=0; i&lt;input_num_values; i++)
<a name="l02221"></a>02221     {
<a name="l02222"></a>02222 
<a name="l02223"></a>02223       new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02224"></a>02224       sscanf(new_line,<span class="stringliteral">&quot;  %le  %le  %le  %le  %le  %le&quot;</span>, &amp;d_energy, &amp;d_rayleigh, &amp;d_compton, &amp;d_photelectric, &amp;d_total_mfp, &amp;d_pmax);
<a name="l02225"></a>02225 
<a name="l02226"></a>02226       <span class="comment">// Find and store the minimum total MFP at the current energy, for every material&#39;s maximum density:</span>
<a name="l02227"></a>02227       <span class="keywordtype">float</span> temp_mfp = d_total_mfp*(density_nominal[mat])/(density_max[mat]);
<a name="l02228"></a>02228       <span class="keywordflow">if</span> (temp_mfp &lt; (*mfp_Woodcock_table_ptr)[i].x)
<a name="l02229"></a>02229         (*mfp_Woodcock_table_ptr)[i].x = temp_mfp;       <span class="comment">// Store minimum total mfp [cm]</span>
<a name="l02230"></a>02230 
<a name="l02231"></a>02231       <span class="comment">// Store the inverse MFP data points with [num_values rows]*[MAX_MATERIALS columns]</span>
<a name="l02232"></a>02232       <span class="comment">// Scaling the table to the nominal density so that I can re-scale in the kernel to the actual local density:</span>
<a name="l02233"></a>02233       (*mfp_table_a_ptr)[i*(<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>)+mat].x = 1.0/(d_total_mfp*density_nominal[mat]);   <span class="comment">// inverse TOTAL mfp * nominal density</span>
<a name="l02234"></a>02234       (*mfp_table_a_ptr)[i*(<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>)+mat].y = 1.0/(d_compton  *density_nominal[mat]);   <span class="comment">// inverse Compton mfp * nominal density</span>
<a name="l02235"></a>02235       (*mfp_table_a_ptr)[i*(<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>)+mat].z = 1.0/(d_rayleigh *density_nominal[mat]);   <span class="comment">// inverse Rayleigh mfp * nominal density</span>
<a name="l02236"></a>02236 
<a name="l02237"></a>02237       rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#a3030ed3f1f07daf452bf53fe6aa8f7a1">pmax</a>[i*(<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>)+mat] = d_pmax;    <span class="comment">// Store the maximum cumulative probability of atomic form factor F^2 for</span>
<a name="l02238"></a>02238 
<a name="l02239"></a>02239       <span class="keywordflow">if</span> (0==i &amp;&amp; 0==mat)
<a name="l02240"></a>02240       {
<a name="l02241"></a>02241         mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>  = d_energy;   <span class="comment">// Store the first energy of the first material</span>
<a name="l02242"></a>02242       }
<a name="l02243"></a>02243 
<a name="l02244"></a>02244       <span class="keywordflow">if</span> (0==i)
<a name="l02245"></a>02245       {
<a name="l02246"></a>02246         <span class="keywordflow">if</span> (fabs(d_energy-mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>)&gt;1.0e-9)
<a name="l02247"></a>02247         {
<a name="l02248"></a>02248           printf(<span class="stringliteral">&quot;\n\n   !!load_material ERROR!! Incorrect first energy value given in material \&#39;%s\&#39;: input=%f, expected=%f\n&quot;</span>, file_name_materials[mat], d_energy, mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>);
<a name="l02249"></a>02249           exit(-2);
<a name="l02250"></a>02250         }
<a name="l02251"></a>02251       }
<a name="l02252"></a>02252       <span class="keywordflow">else</span> <span class="keywordflow">if</span> (1==i)
<a name="l02253"></a>02253       {
<a name="l02254"></a>02254         delta_e = d_energy-e_last;
<a name="l02255"></a>02255       }
<a name="l02256"></a>02256       <span class="keywordflow">else</span> <span class="keywordflow">if</span> (i&gt;1)
<a name="l02257"></a>02257       {
<a name="l02258"></a>02258         <span class="keywordflow">if</span> (((fabs((d_energy-e_last)-delta_e))/delta_e)&gt;0.001)  <span class="comment">// Tolerate up to a 0.1% relative variation in the delta e (for each bin) to account for possible precission errors reading the energy values</span>
<a name="l02259"></a>02259         {
<a name="l02260"></a>02260           printf(<span class="stringliteral">&quot;  !!ERROR reading material data!! The energy step between mean free path values is not constant!!\n      (maybe not enough decimals given for the energy values)\n      #value = %d, First delta: %f , New delta: %f, Energy: %f ; Rel.Dif=%f\n&quot;</span>, i, delta_e, (d_energy-e_last), d_energy,((fabs((d_energy-e_last)-delta_e))/delta_e));
<a name="l02261"></a>02261           exit(-2);
<a name="l02262"></a>02262         }
<a name="l02263"></a>02263       }
<a name="l02264"></a>02264       e_last = d_energy;
<a name="l02265"></a>02265     }
<a name="l02266"></a>02266     
<a name="l02267"></a>02267     <span class="keywordflow">if</span> (0==mat) <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                Lowest energy first bin = %f eV, last bin = %f eV; bin width = %f eV\n&quot;</span>, (mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a>), e_last, delta_e);
<a name="l02268"></a>02268 
<a name="l02269"></a>02269     <span class="comment">// -- Store the inverse of delta energy:</span>
<a name="l02270"></a>02270     mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#a4b743ecf01ca7d075cc6b36b9ce2f0a1">ide</a> = 1.0f/delta_e;
<a name="l02271"></a>02271 
<a name="l02272"></a>02272     <span class="comment">// -- Store MFP data slope &#39;b&#39; (.y for Woodcock):</span>
<a name="l02273"></a>02273     <span class="keywordflow">for</span> (i=0; i&lt;(input_num_values-1); i++)
<a name="l02274"></a>02274     {
<a name="l02275"></a>02275       bin = i*MAX_MATERIALS+mat;                   <span class="comment">// Set current bin, skipping MAX_MATERIALS columns</span>
<a name="l02276"></a>02276       (*mfp_table_b_ptr)[bin].x = ((*mfp_table_a_ptr)[bin+<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>].x - (*mfp_table_a_ptr)[bin].x) / delta_e;
<a name="l02277"></a>02277       (*mfp_table_b_ptr)[bin].y = ((*mfp_table_a_ptr)[bin+<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>].y - (*mfp_table_a_ptr)[bin].y) / delta_e;
<a name="l02278"></a>02278       (*mfp_table_b_ptr)[bin].z = ((*mfp_table_a_ptr)[bin+<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>].z - (*mfp_table_a_ptr)[bin].z) / delta_e;
<a name="l02279"></a>02279     }
<a name="l02280"></a>02280     <span class="comment">// After maximum energy (last bin), assume constant slope:</span>
<a name="l02281"></a>02281     (*mfp_table_b_ptr)[(input_num_values-1)*MAX_MATERIALS+mat] = (*mfp_table_b_ptr)[(input_num_values-2)*MAX_MATERIALS+mat];
<a name="l02282"></a>02282 
<a name="l02283"></a>02283     <span class="comment">// -- Rescale the &#39;a&#39; parameter (.x for Woodcock) as if the bin started at energy = 0: we will not have to rescale to the bin minimum energy every time</span>
<a name="l02284"></a>02284     <span class="keywordflow">for</span> (i=0; i&lt;input_num_values; i++)
<a name="l02285"></a>02285     {
<a name="l02286"></a>02286       d_energy = mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a> + i*delta_e;   <span class="comment">// Set current bin lowest energy value</span>
<a name="l02287"></a>02287       bin = i*MAX_MATERIALS+mat;                   <span class="comment">// Set current bin, skipping MAX_MATERIALS columns</span>
<a name="l02288"></a>02288       (*mfp_table_a_ptr)[bin].x = (*mfp_table_a_ptr)[bin].x - d_energy*(*mfp_table_b_ptr)[bin].x;
<a name="l02289"></a>02289       (*mfp_table_a_ptr)[bin].y = (*mfp_table_a_ptr)[bin].y - d_energy*(*mfp_table_b_ptr)[bin].y;
<a name="l02290"></a>02290       (*mfp_table_a_ptr)[bin].z = (*mfp_table_a_ptr)[bin].z - d_energy*(*mfp_table_b_ptr)[bin].z;
<a name="l02291"></a>02291     }
<a name="l02292"></a>02292 
<a name="l02293"></a>02293     <span class="comment">// -- Reading data for RAYLEIGH INTERACTIONS (RITA sampling  of atomic form factor from EPDL database):</span>
<a name="l02294"></a>02294     <span class="keywordflow">do</span>
<a name="l02295"></a>02295     {
<a name="l02296"></a>02296       new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02297"></a>02297       <span class="keywordflow">if</span> (gzeof(file_ptr)!=0)                           <span class="comment">//  !!zlib!!</span>
<a name="l02298"></a>02298       {
<a name="l02299"></a>02299         printf(<span class="stringliteral">&quot;\n\n   !!End-of-file ERROR!! Rayleigh data not found: \&quot;#[DATA VALUES...\&quot; in file \&#39;%s\&#39;. Last line read: %s\n\n&quot;</span>, file_name_materials[mat], new_line);
<a name="l02300"></a>02300         exit(-2);
<a name="l02301"></a>02301       }
<a name="l02302"></a>02302     }
<a name="l02303"></a>02303     <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;[DATA VALUES&quot;</span>)==NULL);   <span class="comment">// Skip all lines until this text is found</span>
<a name="l02304"></a>02304       
<a name="l02305"></a>02305     new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">// Read the number of data points in Rayleigh     //  !!zlib!! </span>
<a name="l02306"></a>02306     sscanf(new_line, <span class="stringliteral">&quot;# %d&quot;</span>, &amp;input_rayleigh_values);
<a name="l02307"></a>02307         
<a name="l02308"></a>02308     <span class="keywordflow">if</span> (input_rayleigh_values != <a class="code" href="MC-GPU__v1_83_8h.html#a50e5fe2bed6d3ba8c550beb81e5e40e5">NP_RAYLEIGH</a>)
<a name="l02309"></a>02309     {
<a name="l02310"></a>02310       printf(<span class="stringliteral">&quot;\n\n   !!ERROR!! The number of values for Rayleigh sampling is different than the allocated space: input=%d, NP_RAYLEIGH=%d. File=\&#39;%s\&#39;\n&quot;</span>, input_rayleigh_values, <a class="code" href="MC-GPU__v1_83_8h.html#a50e5fe2bed6d3ba8c550beb81e5e40e5">NP_RAYLEIGH</a>, file_name_materials[mat]);
<a name="l02311"></a>02311       exit(-2);
<a name="l02312"></a>02312     }
<a name="l02313"></a>02313     new_line_ptr = gzgets(file_ptr, new_line, 250);    <span class="comment">// Comment line:  #[SAMPLING DATA FROM COMMON/CGRA/: X, P, A, B, ITL, ITU]     //  !!zlib!!</span>
<a name="l02314"></a>02314     <span class="keywordflow">for</span> (i=0; i&lt;input_rayleigh_values; i++)
<a name="l02315"></a>02315     {
<a name="l02316"></a>02316       <span class="keywordtype">int</span> itlco_tmp, ituco_tmp;
<a name="l02317"></a>02317       bin = <a class="code" href="MC-GPU__v1_83_8h.html#a50e5fe2bed6d3ba8c550beb81e5e40e5">NP_RAYLEIGH</a>*mat + i;
<a name="l02318"></a>02318 
<a name="l02319"></a>02319       new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02320"></a>02320       sscanf(new_line,<span class="stringliteral">&quot;  %e  %e  %e  %e  %d  %d&quot;</span>, &amp;(rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#a3d91ed8dc782c166106093ae028f0dd8">xco</a>[bin]), &amp;(rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#a85ba59a95287c8d2ec3895b9ea6f8f43">pco</a>[bin]),
<a name="l02321"></a>02321                                                   &amp;(rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#a2ea1b62a4e4cc5900c060fed2d95fec5">aco</a>[bin]), &amp;(rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#af107a7fbdd989e049ba67b9fc2e614c3">bco</a>[bin]),
<a name="l02322"></a>02322                                                   &amp;itlco_tmp, &amp;ituco_tmp);
<a name="l02323"></a>02323 
<a name="l02324"></a>02324       rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#a81e58bece94cd34133d55960cee9fe73">itlco</a>[bin] = (<span class="keywordtype">unsigned</span> char) itlco_tmp;
<a name="l02325"></a>02325       rayleigh_table_ptr-&gt;<a class="code" href="structrayleigh__struct.html#af8eb91b5cbbc3bdf6d84e3aa9525946a">ituco</a>[bin] = (<span class="keywordtype">unsigned</span> char) ituco_tmp;
<a name="l02326"></a>02326                                                   
<a name="l02327"></a>02327     }
<a name="l02328"></a>02328     <span class="comment">//  printf(&quot;    -- Rayleigh sampling data read. Input values = %d\n&quot;,input_rayleigh_values);</span>
<a name="l02329"></a>02329 
<a name="l02330"></a>02330     <span class="comment">// -- Reading COMPTON INTERACTIONS data (relativistic impulse model with approximated one-electron analytical profiles):</span>
<a name="l02331"></a>02331     <span class="keywordflow">do</span>
<a name="l02332"></a>02332     {
<a name="l02333"></a>02333       new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02334"></a>02334       <span class="keywordflow">if</span> (gzeof(file_ptr)!=0)                           <span class="comment">//  !!zlib!!</span>
<a name="l02335"></a>02335       {
<a name="l02336"></a>02336         printf(<span class="stringliteral">&quot;\n\n   !!End-of-file ERROR!! Compton data not found: \&quot;[NUMBER OF SHELLS]\&quot; in file \&#39;%s\&#39;. Last line read: %s\n\n&quot;</span>, file_name_materials[mat], new_line);
<a name="l02337"></a>02337         exit(-2);
<a name="l02338"></a>02338       }
<a name="l02339"></a>02339     }
<a name="l02340"></a>02340     <span class="keywordflow">while</span>(strstr(new_line,<span class="stringliteral">&quot;[NUMBER OF SHELLS&quot;</span>)==NULL);   <span class="comment">// Skip all lines until this text is found</span>
<a name="l02341"></a>02341     new_line_ptr = gzgets(file_ptr, new_line, 250);
<a name="l02342"></a>02342     sscanf(new_line, <span class="stringliteral">&quot;# %d&quot;</span>, &amp;input_num_shells);      <span class="comment">// Read the NUMBER OF SHELLS</span>
<a name="l02343"></a>02343     <span class="keywordflow">if</span> (input_num_shells&gt;<a class="code" href="MC-GPU__v1_83_8h.html#a7fc9275784608be88cd40ac55f4de283">MAX_SHELLS</a>)
<a name="l02344"></a>02344     {
<a name="l02345"></a>02345       printf(<span class="stringliteral">&quot;\n\n   !!ERROR!! Too many shells for Compton interactions in file \&#39;%s\&#39;: input=%d, MAX_SHELLS=%d\n&quot;</span>, file_name_materials[mat], input_num_shells, <a class="code" href="MC-GPU__v1_83_8h.html#a7fc9275784608be88cd40ac55f4de283">MAX_SHELLS</a>);
<a name="l02346"></a>02346       exit(-2);
<a name="l02347"></a>02347     }
<a name="l02348"></a>02348     compton_table_ptr-&gt;<a class="code" href="structcompton__struct.html#aef81eb007c90d13552ea867912f0d2c3">noscco</a>[mat] = input_num_shells;   <span class="comment">// Store number of shells for this material in structure</span>
<a name="l02349"></a>02349     new_line_ptr = gzgets(file_ptr, new_line, 250);      <span class="comment">// Comment line:  #[SHELL INFORMATION FROM COMMON/CGCO/: FCO, UICO, FJ0, KZCO, KSCO]</span>
<a name="l02350"></a>02350     <span class="keywordtype">int</span> kzco_dummy, ksco_dummy;
<a name="l02351"></a>02351     <span class="keywordflow">for</span> (i=0; i&lt;input_num_shells; i++)
<a name="l02352"></a>02352     {
<a name="l02353"></a>02353 
<a name="l02354"></a>02354       bin = mat + i*<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>;
<a name="l02355"></a>02355 
<a name="l02356"></a>02356       new_line_ptr = gzgets(file_ptr, new_line, 250);   <span class="comment">//  !!zlib!!</span>
<a name="l02357"></a>02357       sscanf(new_line,<span class="stringliteral">&quot; %e  %e  %e  %d  %d&quot;</span>, &amp;(compton_table_ptr-&gt;<a class="code" href="structcompton__struct.html#a181131326c64f04da9eb7f199604371e">fco</a>[bin]), &amp;(compton_table_ptr-&gt;<a class="code" href="structcompton__struct.html#a32dcebc4af75e0620abed9c32ffc2bdd">uico</a>[bin]),
<a name="l02358"></a>02358                                               &amp;(compton_table_ptr-&gt;<a class="code" href="structcompton__struct.html#a2acdfc304123cea06abea723b5b28d38">fj0</a>[bin]), &amp;kzco_dummy, &amp;ksco_dummy);
<a name="l02359"></a>02359     }
<a name="l02360"></a>02360   
<a name="l02361"></a>02361     gzclose(file_ptr);    <span class="comment">// Material data read. Close the current material input file.           //  !!zlib!!</span>
<a name="l02362"></a>02362     
<a name="l02363"></a>02363   }  <span class="comment">// [&quot;for&quot; loop: continue with next material]</span>
<a name="l02364"></a>02364 
<a name="l02365"></a>02365 
<a name="l02366"></a>02366   <span class="comment">// -- Store Woodcock MFP slope in component &#39;.y&#39;:</span>
<a name="l02367"></a>02367   <span class="keywordflow">for</span> (i=0; i&lt;(mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a>-1); i++)
<a name="l02368"></a>02368     (*mfp_Woodcock_table_ptr)[i].y = ((*mfp_Woodcock_table_ptr)[i+1].x - (*mfp_Woodcock_table_ptr)[i].x)/delta_e;
<a name="l02369"></a>02369 
<a name="l02370"></a>02370   <span class="comment">// -- Rescale the first parameter in component .x for Woodcock</span>
<a name="l02371"></a>02371   <span class="keywordflow">for</span> (i=0; i&lt;mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#a2bea2f84374be1b9afd5d896b16a7a46">num_values</a>; i++)
<a name="l02372"></a>02372   {
<a name="l02373"></a>02373     (*mfp_Woodcock_table_ptr)[i].x = (*mfp_Woodcock_table_ptr)[i].x - (mfp_table_data-&gt;<a class="code" href="structlinear__interp.html#ac07eade974afb1b75d16516d1eb55a33">e0</a> + i*delta_e)*(*mfp_Woodcock_table_ptr)[i].y;
<a name="l02374"></a>02374   }
<a name="l02375"></a>02375   
<a name="l02376"></a>02376 }<span class="comment"></span>
<a name="l02377"></a>02377 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02378"></a>02378 <span class="comment"></span>
<a name="l02379"></a>02379 
<a name="l02380"></a>02380 
<a name="l02381"></a>02381 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l02382"></a>02382 <span class="preprocessor"></span><span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02383"></a>02383 <span class="comment"></span><span class="comment">//!  Select and initialize the CUDA-enabled GPU that will be used in the simulation.</span>
<a name="l02384"></a>02384 <span class="comment"></span><span class="comment">//!  Allocates and copies the simulation data in the GPU global and constant memories.</span>
<a name="l02385"></a>02385 <span class="comment"></span><span class="comment">//!</span>
<a name="l02386"></a>02386 <span class="comment"></span><span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02387"></a>02387 <span class="comment"></span><span class="keywordtype">void</span> init_CUDA_device( <span class="keywordtype">int</span>* gpu_id, <span class="keywordtype">int</span> myID, <span class="keywordtype">int</span> numprocs,
<a name="l02388"></a>02388       <span class="comment">/*Variables to GPU constant memory:*/</span> <span class="keyword">struct</span> <a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a>* voxel_data, <span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>* source_data, <span class="keyword">struct</span> <a class="code" href="structsource__energy__struct.html" title="Structure storing the source energy spectrum data to be sampled using the Walker aliasing algorithm...">source_energy_struct</a>* source_energy_data, <span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>* detector_data, <span class="keyword">struct</span> <a class="code" href="structlinear__interp.html" title="Structure with the basic data required by the linear interpolation of the mean free paths: number of ...">linear_interp</a>* mfp_table_data,
<a name="l02389"></a>02389       <span class="comment">/*Variables to GPU global memory:*/</span> <a class="code" href="structfloat2.html">float2</a>* voxel_mat_dens, <a class="code" href="structfloat2.html">float2</a>** voxel_mat_dens_device, <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> voxel_mat_dens_bytes,
<a name="l02390"></a>02390         <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>* image, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>** image_device, <span class="keywordtype">int</span> image_bytes,
<a name="l02391"></a>02391         <a class="code" href="structfloat2.html">float2</a>* mfp_Woodcock_table, <a class="code" href="structfloat2.html">float2</a>** mfp_Woodcock_table_device, <span class="keywordtype">int</span> mfp_Woodcock_table_bytes,
<a name="l02392"></a>02392         <a class="code" href="structfloat3.html">float3</a>* mfp_table_a, <a class="code" href="structfloat3.html">float3</a>* mfp_table_b, <a class="code" href="structfloat3.html">float3</a>** mfp_table_a_device, <a class="code" href="structfloat3.html">float3</a>** mfp_table_b_device, <span class="keywordtype">int</span> mfp_table_bytes,
<a name="l02393"></a>02393         <span class="keyword">struct</span> <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a>* rayleigh_table, <span class="keyword">struct</span> <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a>** rayleigh_table_device,
<a name="l02394"></a>02394         <span class="keyword">struct</span> <a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>* compton_table, <span class="keyword">struct</span> <a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>** compton_table_device, 
<a name="l02395"></a>02395         <span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>** detector_data_device, <span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>** source_data_device,
<a name="l02396"></a>02396         <a class="code" href="structulonglong2.html">ulonglong2</a>* voxels_Edep, <a class="code" href="structulonglong2.html">ulonglong2</a>** voxels_Edep_device, <span class="keywordtype">int</span> voxels_Edep_bytes, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_x_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_x_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_y_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_y_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_z_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span>* dose_ROI_z_max,
<a name="l02397"></a>02397         <a class="code" href="structulonglong2.html">ulonglong2</a>* materials_dose, <a class="code" href="structulonglong2.html">ulonglong2</a>** materials_dose_device, <span class="keywordtype">int</span> flag_material_dose, <span class="keywordtype">int</span> num_projections)
<a name="l02398"></a>02398 {
<a name="l02399"></a>02399   <span class="keywordtype">int</span> deviceCount;
<a name="l02400"></a>02400   checkCudaErrors(cudaGetDeviceCount(&amp;deviceCount));
<a name="l02401"></a>02401   <span class="keywordflow">if</span> (0==deviceCount)
<a name="l02402"></a>02402   {
<a name="l02403"></a>02403     printf(<span class="stringliteral">&quot;\n  !!ERROR!! No CUDA enabled GPU detected by thread #%d!!\n\n&quot;</span>, myID);
<a name="l02404"></a>02404     exit(-1);
<a name="l02405"></a>02405   }  
<a name="l02406"></a>02406   
<a name="l02407"></a>02407   
<a name="l02408"></a>02408 <span class="preprocessor">#ifdef USING_MPI</span>
<a name="l02409"></a>02409 <span class="preprocessor"></span>
<a name="l02410"></a>02410   <span class="comment">// *** Select the appropriate GPUs in the different workstations in the MPI hostfile:</span>
<a name="l02411"></a>02411   <span class="comment">//     The idea is that each threads will wait for the previous thread to send a messages with its processor name and GPU id, </span>
<a name="l02412"></a>02412   <span class="comment">//     then it will assign the current GPU, and finally it will notify the following thread:</span>
<a name="l02413"></a>02413       
<a name="l02414"></a>02414   <span class="keywordflow">if</span> (numprocs&gt;1)
<a name="l02415"></a>02415   {      
<a name="l02416"></a>02416     <span class="keyword">const</span> <span class="keywordtype">int</span> NODE_NAME_LENGTH = 31;
<a name="l02417"></a>02417     <span class="keywordtype">char</span> processor_name[NODE_NAME_LENGTH+1], previous_processor_name[NODE_NAME_LENGTH+1];
<a name="l02418"></a>02418     <span class="keywordtype">int</span> resultlen = -1;
<a name="l02419"></a>02419     
<a name="l02420"></a>02420     MPI_Get_processor_name(processor_name, &amp;resultlen);
<a name="l02421"></a>02421     
<a name="l02422"></a>02422     MPI_Status status;
<a name="l02423"></a>02423     
<a name="l02424"></a>02424     <span class="keywordtype">int</span> gpu_id_to_avoid = *gpu_id;
<a name="l02425"></a>02425 
<a name="l02426"></a>02426     clock_t clock_start;
<a name="l02427"></a>02427     <span class="keywordflow">if</span> (myID == (numprocs-1))
<a name="l02428"></a>02428       clock_start = clock();
<a name="l02429"></a>02429         
<a name="l02430"></a>02430 
<a name="l02431"></a>02431     <span class="comment">// Unless we are the first thread, wait for a message from the previous thread:</span>
<a name="l02432"></a>02432     <span class="comment">// The MPI_Recv command will block the execution of the code until the previous threads have communicated and shared the appropriate information.</span>
<a name="l02433"></a>02433     <span class="keywordflow">if</span> (0!=myID)
<a name="l02434"></a>02434     {     
<a name="l02435"></a>02435       MPI_Recv(previous_processor_name, NODE_NAME_LENGTH, MPI_CHAR, myID-1, 111, MPI_COMM_WORLD, &amp;status);   <span class="comment">// Receive the processor name and gpu_id from the previous thread</span>
<a name="l02436"></a>02436           <span class="comment">// printf(&quot;\n -&gt; MPI_Recv thread %d: gpu_id=%d, %s\n&quot;, myID, (int)previous_processor_name[NODE_NAME_LENGTH-1], previous_processor_name); fflush(stdout);  //!!Verbose!! </span>
<a name="l02437"></a>02437     }
<a name="l02438"></a>02438     
<a name="l02439"></a>02439 
<a name="l02440"></a>02440     <span class="comment">// Compare the 30 first characters of the 2 names to see if we changed the node, except for the first thread that allways gets GPU 0:</span>
<a name="l02441"></a>02441     <span class="keywordflow">if</span> ((0==myID) || (0!=strncmp(processor_name, previous_processor_name, NODE_NAME_LENGTH-1)))
<a name="l02442"></a>02442     { 
<a name="l02443"></a>02443       *gpu_id = 0;    <span class="comment">// Thread in a new node: assign to GPU 0:</span>
<a name="l02444"></a>02444     }
<a name="l02445"></a>02445     <span class="keywordflow">else</span>
<a name="l02446"></a>02446     {
<a name="l02447"></a>02447       <span class="comment">// Current thread in the same node as the previous one: assign next GPU (previous GPU id given in element NODE_NAME_LENGTH-1 of the array)</span>
<a name="l02448"></a>02448       *gpu_id = (int)previous_processor_name[NODE_NAME_LENGTH-1] + 1;
<a name="l02449"></a>02449     }
<a name="l02450"></a>02450 
<a name="l02451"></a>02451     <span class="comment">// Set the following GPU if this is the one to be skipped (given in the input file):</span>
<a name="l02452"></a>02452     <span class="keywordflow">if</span> (*gpu_id == gpu_id_to_avoid)
<a name="l02453"></a>02453     {
<a name="l02454"></a>02454       *gpu_id = *gpu_id + 1;  
<a name="l02455"></a>02455       printf(<span class="stringliteral">&quot;             Skipping GPU %d in thread %d (%s), as selected in the input file: gpu_id=%d\n&quot;</span>, gpu_id_to_avoid, myID, processor_name, *gpu_id); fflush(stdout);
<a name="l02456"></a>02456     }
<a name="l02457"></a>02457     
<a name="l02458"></a>02458     
<a name="l02459"></a>02459     <span class="comment">// Send the processor and GPU id to the following thread, unless we are the last thread:</span>
<a name="l02460"></a>02460     <span class="keywordflow">if</span> (myID != (numprocs-1))
<a name="l02461"></a>02461     { 
<a name="l02462"></a>02462       processor_name[NODE_NAME_LENGTH-1] = (char)(*gpu_id);  <span class="comment">// Store GPU number in the last element of the array</span>
<a name="l02463"></a>02463       
<a name="l02464"></a>02464           <span class="comment">// printf(&quot; &lt;- MPI_Send thread %d: gpu_id=%d, %s\n&quot;, myID, (int)processor_name[NODE_NAME_LENGTH-1], processor_name); fflush(stdout);  //!!Verbose!!</span>
<a name="l02465"></a>02465       MPI_Send(processor_name, NODE_NAME_LENGTH, MPI_CHAR, myID+1, 111, MPI_COMM_WORLD);  <span class="comment">// Send processor name and gpu_id to the following thread (tag is the current thread id)</span>
<a name="l02466"></a>02466     }
<a name="l02467"></a>02467     <span class="keywordflow">else</span>
<a name="l02468"></a>02468     {
<a name="l02469"></a>02469       printf(<span class="stringliteral">&quot;           -- Time spent communicating between threads to determine the GPU id to use in each thread: %.6f s\n&quot;</span>, ((<span class="keywordtype">double</span>)(clock()-clock_start))/CLOCKS_PER_SEC); fflush(stdout);
<a name="l02470"></a>02470     }
<a name="l02471"></a>02471     
<a name="l02472"></a>02472   }
<a name="l02473"></a>02473   
<a name="l02474"></a>02474 <span class="preprocessor">#endif  </span>
<a name="l02475"></a>02475 <span class="preprocessor"></span>  
<a name="l02476"></a>02476   <span class="keywordflow">if</span> (*gpu_id&gt;=deviceCount)
<a name="l02477"></a>02477   {
<a name="l02478"></a>02478     printf(<span class="stringliteral">&quot;\n\n  !!WARNING!! The selected GPU number is too high, this device number does not exist!! GPU_id (starting at 0)=%d, deviceCount=%d\n&quot;</span>, (*gpu_id), deviceCount); fflush(stdout);
<a name="l02479"></a>02479     <span class="keywordflow">if</span> (numprocs==1)
<a name="l02480"></a>02480     {
<a name="l02481"></a>02481       *gpu_id = gpuGetMaxGflopsDeviceId();
<a name="l02482"></a>02482       printf(<span class="stringliteral">&quot;            Selecting the fastest GPU available using gpuGetMaxGflopsDeviceId(): GPU_id = %d\n\n&quot;</span>, (*gpu_id)); fflush(stdout);
<a name="l02483"></a>02483     }    
<a name="l02484"></a>02484     <span class="keywordflow">else</span>
<a name="l02485"></a>02485     {
<a name="l02486"></a>02486       exit(-1);    
<a name="l02487"></a>02487     }
<a name="l02488"></a>02488   }     
<a name="l02489"></a>02489 
<a name="l02490"></a>02490   checkCudaErrors(cudaSetDevice(*gpu_id));   <span class="comment">// Set the GPU device. (optionally use: cutGetMaxGflopsDeviceId())</span>
<a name="l02491"></a>02491    
<a name="l02492"></a>02492   
<a name="l02493"></a>02493   cudaDeviceProp deviceProp;
<a name="l02494"></a>02494   checkCudaErrors(cudaGetDeviceProperties(&amp;deviceProp, *gpu_id));
<a name="l02495"></a>02495   <span class="keywordflow">if</span> (deviceProp.major&gt;99 || deviceProp.minor&gt;99)
<a name="l02496"></a>02496   {
<a name="l02497"></a>02497     printf(<span class="stringliteral">&quot;\n\n\n  !!ERROR!! The selected GPU device does not support CUDA!! GPU_id=%d, deviceCount=%d, compute capability=%d.%d\n\n\n&quot;</span>, (*gpu_id), deviceCount, deviceProp.major,deviceProp.minor);
<a name="l02498"></a>02498     exit(-1);
<a name="l02499"></a>02499   }
<a name="l02500"></a>02500 
<a name="l02501"></a>02501   <span class="keywordflow">if</span> (deviceProp.major&gt;1)
<a name="l02502"></a>02502   {
<a name="l02503"></a>02503     
<a name="l02504"></a>02504 <span class="preprocessor">#ifdef LARGE_CACHE  </span>
<a name="l02505"></a>02505 <span class="preprocessor"></span>    <span class="comment">// -- Compute capability &gt; 1: set a large L1 cache for the global memory, reducing the size of the shared memory:</span>
<a name="l02506"></a>02506     <span class="comment">//       cudaFuncCachePreferShared: shared memory is 48 KB</span>
<a name="l02507"></a>02507     <span class="comment">//       cudaFuncCachePreferL1: shared memory is 16 KB</span>
<a name="l02508"></a>02508     <span class="comment">//       cudaFuncCachePreferNone: no preference</span>
<a name="l02509"></a>02509     printf(<span class="stringliteral">&quot;\n       ==&gt; CUDA: LARGE_CACHE defined --&gt; setting a large global memory cache (L1) and a small shared memory (cudaFuncCachePreferL1).\n&quot;</span>);
<a name="l02510"></a>02510     cudaFuncSetCacheConfig(<a class="code" href="MC-GPU__kernel__v1_83_8cu.html#ab1e2e311c809fe74d093498e53c9c954" title="Initialize the image array, ie, set all pixels to zero Essentially, this function has the same effect...">track_particles</a>, cudaFuncCachePreferL1);            <span class="comment">// -- Set a large cache instead of a large shared memory.</span>
<a name="l02511"></a>02511         <span class="comment">// #else</span>
<a name="l02512"></a>02512         <span class="comment">// -- Using default:</span>
<a name="l02513"></a>02513         <span class="comment">// printf(&quot;\n       ==&gt; CUDA: LARGE_CACHE not defined --&gt; setting a large shared memory and a small global memory cache (cudaFuncCachePreferShared).\n&quot;);</span>
<a name="l02514"></a>02514         <span class="comment">//    cudaFuncSetCacheConfig(track_particles, cudaFuncCachePreferShared);            // !!DeBuG!! Setting size of shared memory/global cache</span>
<a name="l02515"></a>02515 <span class="preprocessor">#endif</span>
<a name="l02516"></a>02516 <span class="preprocessor"></span>
<a name="l02517"></a>02517   }
<a name="l02518"></a>02518 
<a name="l02519"></a>02519   <span class="keyword">register</span> <span class="keywordtype">int</span> GPU_cores = _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * deviceProp.multiProcessorCount;    <span class="comment">// CUDA SDK function to get the number of GPU cores</span>
<a name="l02520"></a>02520 
<a name="l02521"></a>02521   <span class="comment">// -- Reading the device properties:</span>
<a name="l02522"></a>02522   
<a name="l02523"></a>02523 <span class="preprocessor">#ifdef USING_MPI   </span>
<a name="l02524"></a>02524 <span class="preprocessor"></span>  printf(<span class="stringliteral">&quot;\n       ==&gt; CUDA (MPI process #%d): %d CUDA enabled GPU detected! Using device #%d: \&quot;%s\&quot;\n&quot;</span>, myID, deviceCount, (*gpu_id), deviceProp.name);    
<a name="l02525"></a>02525 <span class="preprocessor">#else  </span>
<a name="l02526"></a>02526 <span class="preprocessor"></span>  printf(<span class="stringliteral">&quot;\n       ==&gt; CUDA: %d CUDA enabled GPU detected! Using device #%d: \&quot;%s\&quot;\n&quot;</span>, deviceCount, (*gpu_id), deviceProp.name);    
<a name="l02527"></a>02527 <span class="preprocessor">#endif</span>
<a name="l02528"></a>02528 <span class="preprocessor"></span>  printf(<span class="stringliteral">&quot;                 Compute capability: %d.%d, Number multiprocessors: %d, Number cores: %d\n&quot;</span>, deviceProp.major, deviceProp.minor, deviceProp.multiProcessorCount, GPU_cores);
<a name="l02529"></a>02529   printf(<span class="stringliteral">&quot;                 Clock rate: %.2f GHz, Global memory: %.3f Mbyte, Constant memory: %.2f kbyte\n&quot;</span>, deviceProp.clockRate*1.0e-6f, deviceProp.totalGlobalMem/(1024.f*1024.f), deviceProp.totalConstMem/1024.f);
<a name="l02530"></a>02530   printf(<span class="stringliteral">&quot;                 Shared memory per block: %.2f kbyte, Registers per block: %.2f kbyte\n&quot;</span>, deviceProp.sharedMemPerBlock/1024.f, deviceProp.regsPerBlock/1024.f);
<a name="l02531"></a>02531   <span class="keywordtype">int</span> driverVersion = 0, runtimeVersion = 0;  
<a name="l02532"></a>02532   cudaDriverGetVersion(&amp;driverVersion);
<a name="l02533"></a>02533   cudaRuntimeGetVersion(&amp;runtimeVersion);
<a name="l02534"></a>02534   printf(<span class="stringliteral">&quot;                 CUDA Driver Version: %d.%d, Runtime Version: %d.%d\n&quot;</span>, driverVersion/1000, driverVersion%100, runtimeVersion/1000, runtimeVersion%100);
<a name="l02535"></a>02535 
<a name="l02536"></a>02536   <span class="keywordflow">if</span> (0!=deviceProp.kernelExecTimeoutEnabled)
<a name="l02537"></a>02537   {
<a name="l02538"></a>02538     printf(<span class="stringliteral">&quot;\n\n\n   !!WARNING!! The selected GPU is connected to a display and therefore CUDA driver will limit the kernel run time to 5 seconds and the simulation will likely fail!!\n&quot;</span>);
<a name="l02539"></a>02539     printf( <span class="stringliteral">&quot;              You can fix this by executing the simulation in a different GPU (select number in the input file) or by turning off the window manager and using the text-only Linux shell.\n\n\n&quot;</span>);
<a name="l02540"></a>02540     <span class="comment">// exit(-1);</span>
<a name="l02541"></a>02541   }    
<a name="l02542"></a>02542 
<a name="l02543"></a>02543   clock_t clock_init = clock();    
<a name="l02544"></a>02544 
<a name="l02545"></a>02545   <span class="comment">// -- Allocate the constant variables in the device:</span>
<a name="l02546"></a>02546   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#a129396d7723d11e10c673c1171773cdc" title="Global variable to be stored in the GPU constant memory defining the size of the voxel phantom...">voxel_data_CONST</a>,     voxel_data,     <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a>)));
<a name="l02547"></a>02547   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#ad18b13e5db1d8c7c4acdd9af492d6914" title="Global variable to be stored in the GPU constant memory defining the source energy spectrum...">source_energy_data_CONST</a>, source_energy_data, <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structsource__energy__struct.html" title="Structure storing the source energy spectrum data to be sampled using the Walker aliasing algorithm...">source_energy_struct</a>)));
<a name="l02548"></a>02548   
<a name="l02549"></a>02549 <span class="comment">// Source, detector data now copied to global memory and transfered to shared memory in the kernel. OLD CODE:  checkCudaErrors(cudaMemcpyToSymbol(detector_data_CONST,  detector_data,  sizeof(struct detector_struct)));</span>
<a name="l02550"></a>02550   
<a name="l02551"></a>02551   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#a8a3e410fb23d04f8a27d3d71c2381d24" title="Global variable to be stored in the GPU constant memory defining the linear interpolation data...">mfp_table_data_CONST</a>, mfp_table_data, <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structlinear__interp.html" title="Structure with the basic data required by the linear interpolation of the mean free paths: number of ...">linear_interp</a>)));
<a name="l02552"></a>02552 
<a name="l02553"></a>02553   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#a5d34c13b457e76dec83755628ead56fc" title="Global variable to be stored in the GPU constant memory defining the coordinates of the dose depositi...">dose_ROI_x_min_CONST</a>, dose_ROI_x_min, <span class="keyword">sizeof</span>(<span class="keywordtype">short</span> <span class="keywordtype">int</span>)));
<a name="l02554"></a>02554   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#a3c22177ce0c73c17ca8716d997678b61">dose_ROI_x_max_CONST</a>, dose_ROI_x_max, <span class="keyword">sizeof</span>(<span class="keywordtype">short</span> <span class="keywordtype">int</span>)));
<a name="l02555"></a>02555   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#ac161c8f74fd460e82e73917b3afff5fc">dose_ROI_y_min_CONST</a>, dose_ROI_y_min, <span class="keyword">sizeof</span>(<span class="keywordtype">short</span> <span class="keywordtype">int</span>)));
<a name="l02556"></a>02556   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#ab591ece7d8e024dcc8de71f59bc1cea5">dose_ROI_y_max_CONST</a>, dose_ROI_y_max, <span class="keyword">sizeof</span>(<span class="keywordtype">short</span> <span class="keywordtype">int</span>)));
<a name="l02557"></a>02557   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#a81db18966d58018976252f898cd905d9">dose_ROI_z_min_CONST</a>, dose_ROI_z_min, <span class="keyword">sizeof</span>(<span class="keywordtype">short</span> <span class="keywordtype">int</span>)));
<a name="l02558"></a>02558   checkCudaErrors(cudaMemcpyToSymbol(<a class="code" href="MC-GPU__v1_83_8h.html#aea8fcb598ccc9c18ae601f9c7d936b9d">dose_ROI_z_max_CONST</a>, dose_ROI_z_max, <span class="keyword">sizeof</span>(<span class="keywordtype">short</span> <span class="keywordtype">int</span>)));
<a name="l02559"></a>02559   
<a name="l02560"></a>02560 
<a name="l02561"></a>02561 
<a name="l02562"></a>02562   <span class="keywordtype">double</span> total_mem = <span class="keyword">sizeof</span>(<span class="keyword">struct </span><a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a>)+sizeof(struct <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>)+sizeof(struct <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>)+sizeof(struct <a class="code" href="structlinear__interp.html" title="Structure with the basic data required by the linear interpolation of the mean free paths: number of ...">linear_interp</a>) + 6*<span class="keyword">sizeof</span>(<span class="keywordtype">short</span> int);
<a name="l02563"></a>02563   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Constant data successfully copied to the device. CONSTANT memory used: %lf kbytes (%.1lf%%)\n&quot;</span>, total_mem/1024.0, 100.0*total_mem/deviceProp.totalConstMem);
<a name="l02564"></a>02564   
<a name="l02565"></a>02565 
<a name="l02566"></a>02566   <span class="comment">// -- Allocate the device global memory:</span>
<a name="l02567"></a>02567 
<a name="l02568"></a>02568   <span class="keywordflow">if</span> (*dose_ROI_x_max &gt; -1)  <span class="comment">// Allocate dose array only if the tally is not disabled</span>
<a name="l02569"></a>02569   {
<a name="l02570"></a>02570     checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) voxels_Edep_device, voxels_Edep_bytes));
<a name="l02571"></a>02571     <span class="keywordflow">if</span> (*voxels_Edep_device==NULL)
<a name="l02572"></a>02572     {
<a name="l02573"></a>02573       printf(<span class="stringliteral">&quot;\n cudaMalloc ERROR!! Error allocating the dose array on the device global memory!! (%lf Mbytes)\n&quot;</span>, voxels_Edep_bytes/(1024.0*1024.0));
<a name="l02574"></a>02574       exit(-1);
<a name="l02575"></a>02575     }
<a name="l02576"></a>02576   }
<a name="l02577"></a>02577   
<a name="l02578"></a>02578   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) voxel_mat_dens_device, voxel_mat_dens_bytes));
<a name="l02579"></a>02579   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) image_device,          image_bytes));
<a name="l02580"></a>02580   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) mfp_Woodcock_table_device, mfp_Woodcock_table_bytes));
<a name="l02581"></a>02581   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) mfp_table_a_device,    mfp_table_bytes));
<a name="l02582"></a>02582   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) mfp_table_b_device,    mfp_table_bytes));
<a name="l02583"></a>02583   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) rayleigh_table_device, <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a>)));
<a name="l02584"></a>02584   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) compton_table_device,  <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>))); 
<a name="l02585"></a>02585   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) detector_data_device,  num_projections*<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>)));
<a name="l02586"></a>02586   checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) source_data_device,    num_projections*<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>)));    <span class="comment">// The array of detectors, sources has &quot;MAX_NUM_PROJECTIONS&quot; elements but I am allocating only the used &quot;num_projections&quot; elements to the GPU</span>
<a name="l02587"></a>02587   
<a name="l02588"></a>02588   <span class="keywordflow">if</span> (flag_material_dose==1)
<a name="l02589"></a>02589     checkCudaErrors(cudaMalloc((<span class="keywordtype">void</span>**) materials_dose_device, <a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>*<span class="keyword">sizeof</span>(<a class="code" href="structulonglong2.html">ulonglong2</a>)));    <span class="comment">// !!tally_materials_dose!!</span>
<a name="l02590"></a>02590   
<a name="l02591"></a>02591   total_mem = voxels_Edep_bytes + voxel_mat_dens_bytes + image_bytes + mfp_Woodcock_table_bytes + 2*mfp_table_bytes + <span class="keyword">sizeof</span>(<span class="keyword">struct </span><a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>) + sizeof(struct <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a>) + num_projections*(<span class="keyword">sizeof</span>(<span class="keyword">struct </span><a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>) + sizeof(struct <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>));
<a name="l02592"></a>02592   <span class="keywordflow">if</span> (*voxel_mat_dens_device==NULL || *image_device==NULL || *mfp_Woodcock_table_device==NULL || *mfp_table_a_device==NULL ||
<a name="l02593"></a>02593       *mfp_table_a_device==NULL || *rayleigh_table_device==NULL || *compton_table_device==NULL || *detector_data_device==NULL || *source_data_device==NULL)
<a name="l02594"></a>02594   {
<a name="l02595"></a>02595     printf(<span class="stringliteral">&quot;\n cudaMalloc ERROR!! Device global memory not correctly allocated!! (%lf Mbytes)\n&quot;</span>, total_mem/(1024.0*1024.0));
<a name="l02596"></a>02596     exit(-1);
<a name="l02597"></a>02597   }
<a name="l02598"></a>02598   <span class="keywordflow">else</span>
<a name="l02599"></a>02599   {
<a name="l02600"></a>02600     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Device global memory correctly allocated. GLOBAL memory used: %lf Mbytes (%.1lf%%)\n&quot;</span>, total_mem/(1024.0*1024.0), 100.0*total_mem/deviceProp.totalGlobalMem);
<a name="l02601"></a>02601   }
<a name="l02602"></a>02602 
<a name="l02603"></a>02603   <span class="comment">// --Copy the host memory to the device:</span>
<a name="l02604"></a>02604   checkCudaErrors(cudaMemcpy(*voxel_mat_dens_device, voxel_mat_dens, voxel_mat_dens_bytes,                          cudaMemcpyHostToDevice));
<a name="l02605"></a>02605   checkCudaErrors(cudaMemcpy(*mfp_Woodcock_table_device, mfp_Woodcock_table, mfp_Woodcock_table_bytes,              cudaMemcpyHostToDevice));
<a name="l02606"></a>02606   checkCudaErrors(cudaMemcpy(*mfp_table_a_device,    mfp_table_a,    mfp_table_bytes,                               cudaMemcpyHostToDevice));
<a name="l02607"></a>02607   checkCudaErrors(cudaMemcpy(*mfp_table_b_device,    mfp_table_b,    mfp_table_bytes,                               cudaMemcpyHostToDevice));
<a name="l02608"></a>02608   checkCudaErrors(cudaMemcpy(*rayleigh_table_device, rayleigh_table, <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structrayleigh__struct.html" title="Structure storing the data of the Rayleigh interaction sampling model (equivalent to PENELOPE&#39;s commo...">rayleigh_struct</a>),                cudaMemcpyHostToDevice));
<a name="l02609"></a>02609   checkCudaErrors(cudaMemcpy(*compton_table_device,  compton_table,  <span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structcompton__struct.html" title="Structure storing the data of the Compton interaction sampling model (equivalent to PENELOPE&#39;s common...">compton_struct</a>),                 cudaMemcpyHostToDevice));  
<a name="l02610"></a>02610   checkCudaErrors(cudaMemcpy(*detector_data_device,  detector_data,  num_projections*<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>),cudaMemcpyHostToDevice));
<a name="l02611"></a>02611   checkCudaErrors(cudaMemcpy(*source_data_device,    source_data,    num_projections*<span class="keyword">sizeof</span>(<span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>),  cudaMemcpyHostToDevice));  
<a name="l02612"></a>02612   
<a name="l02613"></a>02613 
<a name="l02614"></a>02614   <span class="comment">//   --Init the image array to 0 using a GPU kernel instead of cudaMemcpy:</span>
<a name="l02615"></a>02615   <span class="comment">//     Simple version: checkCudaErrors( cudaMemcpy( image_device, image, image_bytes, cudaMemcpyHostToDevice) );</span>
<a name="l02616"></a>02616 
<a name="l02617"></a>02617   <span class="keywordtype">int</span> pixels_per_image = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a> * detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>;
<a name="l02618"></a>02618   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Launching kernel to initialize the device image to 0: number of blocks = %d, threads per block = 128\n&quot;</span>, (<span class="keywordtype">int</span>)(ceil(pixels_per_image/128.0f)+0.01f) );
<a name="l02619"></a>02619 
<a name="l02620"></a>02620   init_image_array_GPU&lt;&lt;&lt;(int)(ceil(pixels_per_image/128.0f)+0.01f),128&gt;&gt;&gt;(*image_device, pixels_per_image);
<a name="l02621"></a>02621     cudaThreadSynchronize();      <span class="comment">// Force the runtime to wait until all device tasks have completed</span>
<a name="l02622"></a>02622     getLastCudaError(<span class="stringliteral">&quot;\n\n !!Kernel execution failed initializing the image array!! &quot;</span>);  <span class="comment">// Check if kernel execution generated any error:</span>
<a name="l02623"></a>02623 
<a name="l02624"></a>02624 
<a name="l02625"></a>02625   <span class="comment">//   --Init the dose array to 0 using a GPU kernel, if the tally is not disabled:</span>
<a name="l02626"></a>02626   <span class="keywordflow">if</span> (*dose_ROI_x_max &gt; -1)
<a name="l02627"></a>02627   {      
<a name="l02628"></a>02628     
<a name="l02629"></a>02629     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;       ==&gt; CUDA: Initialize the device dose deposition to 0 using cudaMemcpy.\n&quot;</span>);
<a name="l02630"></a>02630     checkCudaErrors(cudaMemcpy(*voxels_Edep_device, voxels_Edep, voxels_Edep_bytes, cudaMemcpyHostToDevice) );
<a name="l02631"></a>02631    
<a name="l02632"></a>02632 <span class="comment">/*  // -- OPTIONAL CODE: Launch kernel to initialize the device dose deposition to 0 (MAY FAIL IF DOSE MATRIX IS TOO BIG!)    !!DeBuG!!</span>
<a name="l02633"></a>02633 <span class="comment">    int num_voxels_dose = voxels_Edep_bytes/sizeof(ulonglong2);   // Calculate the number of voxels in the dose array</span>
<a name="l02634"></a>02634 <span class="comment">    int num_blocks, num_threads_block = 0;  </span>
<a name="l02635"></a>02635 <span class="comment">    // Select the number of threads per block making sure we don&#39;t try to launch more blocks than CUDA&#39;s maximum value:</span>
<a name="l02636"></a>02636 <span class="comment">    do</span>
<a name="l02637"></a>02637 <span class="comment">    {</span>
<a name="l02638"></a>02638 <span class="comment">      num_threads_block += 64;</span>
<a name="l02639"></a>02639 <span class="comment">      num_blocks = (int)(ceil(((double)num_voxels_dose)/((double)num_threads_block))+0.001);</span>
<a name="l02640"></a>02640 <span class="comment">    }</span>
<a name="l02641"></a>02641 <span class="comment">    while (num_blocks &gt; 65500);    </span>
<a name="l02642"></a>02642 <span class="comment">    MASTER_THREAD printf(&quot;       ==&gt; CUDA: Launching kernel to initialize the device dose deposition to 0: number of blocks = %d, threads per block = %d\n&quot;, num_blocks, num_threads_block);  </span>
<a name="l02643"></a>02643 <span class="comment">    init_dose_array_GPU&lt;&lt;&lt;num_blocks,num_threads_block&gt;&gt;&gt;(*voxels_Edep_device, num_voxels_dose);    </span>
<a name="l02644"></a>02644 <span class="comment">      cudaThreadSynchronize();</span>
<a name="l02645"></a>02645 <span class="comment">      getLastCudaError(&quot;\n\n !!Kernel execution failed initializing the dose array!! &quot;);  // Check if kernel execution generated any error:</span>
<a name="l02646"></a>02646 <span class="comment">*/</span>
<a name="l02647"></a>02647 
<a name="l02648"></a>02648   }
<a name="l02649"></a>02649   
<a name="l02650"></a>02650   <span class="comment">// Init materials_dose array in GPU with 0 (same as host):</span>
<a name="l02651"></a>02651   <span class="keywordflow">if</span> (flag_material_dose==1)
<a name="l02652"></a>02652     checkCudaErrors(cudaMemcpy(*materials_dose_device, materials_dose, <a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>*<span class="keyword">sizeof</span>(<a class="code" href="structulonglong2.html">ulonglong2</a>), cudaMemcpyHostToDevice));   <span class="comment">// !!tally_materials_dose!!</span>
<a name="l02653"></a>02653   
<a name="l02654"></a>02654   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                 Time spent allocating and copying memory to the device: %.6f s\n&quot;</span>, <span class="keywordtype">float</span>(clock()-clock_init)/CLOCKS_PER_SEC);    
<a name="l02655"></a>02655 
<a name="l02656"></a>02656 }
<a name="l02657"></a>02657 
<a name="l02658"></a>02658 <span class="comment"></span>
<a name="l02659"></a>02659 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02660"></a>02660 <span class="comment"></span><span class="comment"></span>
<a name="l02661"></a>02661 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02662"></a>02662 <span class="comment">//! Guestimate a good number of blocks to estimate the speed of different generations </span>
<a name="l02663"></a>02663 <span class="comment">//! of GPUs. Slower GPUs will simulate less particles and hopefully the fastest GPUs </span>
<a name="l02664"></a>02664 <span class="comment">//! will not have to wait much. If the speed is not accurately estimated in the speed test</span>
<a name="l02665"></a>02665 <span class="comment">//! some GPUs will simulate longer than others and valuable simulation time will be wasted </span>
<a name="l02666"></a>02666 <span class="comment">//! in the idle GPUs.</span>
<a name="l02667"></a>02667 <span class="comment">//!</span>
<a name="l02668"></a>02668 <span class="comment">//! In this function the &quot;optimum&quot; number of blocks for the speed test is heuristically </span>
<a name="l02669"></a>02669 <span class="comment">//! computed as the product of three GPU characteristics:</span>
<a name="l02670"></a>02670 <span class="comment">//!   [2.0] * [number of GPU cores] * [core frequency] * [major CUDA compute capability] + [100]</span>
<a name="l02671"></a>02671 <span class="comment">//!</span>
<a name="l02672"></a>02672 <span class="comment">//! The factor 2.0 is arbitrary and can be modified depending on the case (for short </span>
<a name="l02673"></a>02673 <span class="comment">//! simulations this value may have to be reduced or the speed test will take longer </span>
<a name="l02674"></a>02674 <span class="comment">//! than the whole simulation). The constant 100 blocks are added to try to get enough </span>
<a name="l02675"></a>02675 <span class="comment">//! blocks for a reliable timing of slow GPUs.</span>
<a name="l02676"></a>02676 <span class="comment">//!</span>
<a name="l02677"></a>02677 <span class="comment">//! For example, an NVIDIA GeForce 290 will get:</span>
<a name="l02678"></a>02678 <span class="comment">//!   2.0 * 240 (cores) * 1.24 (GHz) * 1 (major compute capability) + 100 =  695.2 ~  695 blocks</span>
<a name="l02679"></a>02679 <span class="comment">//! An NVIDIA GeForce 580 will get:</span>
<a name="l02680"></a>02680 <span class="comment">//!   2.0 * 512 (cores) * 1.54 (GHz) * 2 (major compute capability) + 100 = 3253.9 ~ 3254 blocks </span>
<a name="l02681"></a>02681 <span class="comment">//! In total the 580 gets 5.7 times more blocks than the 290.</span>
<a name="l02682"></a>02682 <span class="comment">//!</span>
<a name="l02683"></a>02683 <span class="comment">//!       @param[in] gpu_id   GPU number</span>
<a name="l02684"></a>02684 <span class="comment">//!       @param[out] num_blocks   Returns a number of blocks related to the expected GPU speed</span>
<a name="l02685"></a>02685 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02686"></a>02686 <span class="comment"></span><span class="keywordtype">int</span> guestimate_GPU_performance(<span class="keywordtype">int</span> gpu_id)
<a name="l02687"></a>02687 {          
<a name="l02688"></a>02688   cudaDeviceProp deviceProp;
<a name="l02689"></a>02689   cudaGetDeviceProperties(&amp;deviceProp, gpu_id);
<a name="l02690"></a>02690   <span class="keywordtype">float</span> num_cores       = (float) _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * deviceProp.multiProcessorCount;
<a name="l02691"></a>02691   <span class="keywordtype">float</span> comp_capability = (float) deviceProp.major;
<a name="l02692"></a>02692   <span class="keywordtype">float</span> frequency       = deviceProp.clockRate*1.0e-6f;
<a name="l02693"></a>02693   
<a name="l02694"></a>02694   return (<span class="keywordtype">int</span>)(2.0f*num_cores*frequency*comp_capability + 100.0f + 0.50f);
<a name="l02695"></a>02695 }
<a name="l02696"></a>02696   
<a name="l02697"></a>02697   
<a name="l02698"></a>02698 <span class="preprocessor">#endif</span>
<a name="l02699"></a>02699 <span class="preprocessor"></span><span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02700"></a>02700 <span class="comment"></span>
<a name="l02701"></a>02701 
<a name="l02702"></a>02702 
<a name="l02703"></a>02703 <span class="comment"></span>
<a name="l02704"></a>02704 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02705"></a>02705 <span class="comment">//! Report the tallied image in ASCII and binary form (32-bit floats).</span>
<a name="l02706"></a>02706 <span class="comment">//! Separate images for primary and scatter radiation are generated.</span>
<a name="l02707"></a>02707 <span class="comment">//! </span>
<a name="l02708"></a>02708 <span class="comment">//!</span>
<a name="l02709"></a>02709 <span class="comment">//!       @param[in] file_name_output   File where tallied image is reported</span>
<a name="l02710"></a>02710 <span class="comment">//!       @param[in] detector_data   Detector description read from the input file (pointer to detector_struct)</span>
<a name="l02711"></a>02711 <span class="comment">//!       @param[in] image  Tallied image (in meV per pixel)</span>
<a name="l02712"></a>02712 <span class="comment">//!       @param[in] time_elapsed   Time elapsed during the main loop execution (in seconds)</span>
<a name="l02713"></a>02713 <span class="comment">//!       @param[in] total_histories   Total number of x-rays simulated</span>
<a name="l02714"></a>02714 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02715"></a><a class="code" href="MC-GPU__v1_83_8h.html#aa23f71412757b1d0e1d4c00e6b8a0c18">02715</a> <span class="comment"></span><span class="keywordtype">int</span> <a class="code" href="MC-GPU__v1_83_8cu.html#aa23f71412757b1d0e1d4c00e6b8a0c18" title="Report the tallied image in ASCII and binary form (32-bit floats).">report_image</a>(<span class="keywordtype">char</span>* file_name_output, <span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>* detector_data, <span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>* source_data, <span class="keywordtype">float</span> mean_energy_spectrum, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>* image, <span class="keywordtype">double</span> time_elapsed, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories, <span class="keywordtype">int</span> current_projection, <span class="keywordtype">int</span> num_projections, <span class="keywordtype">double</span> D_angle, <span class="keywordtype">double</span> initial_angle, <span class="keywordtype">int</span> myID, <span class="keywordtype">int</span> numprocs)
<a name="l02716"></a>02716 {
<a name="l02717"></a>02717   
<a name="l02718"></a>02718   <span class="comment">//  -Find current angle</span>
<a name="l02719"></a>02719   <span class="keywordtype">double</span> current_angle = initial_angle+current_projection*D_angle;
<a name="l02720"></a>02720 
<a name="l02721"></a>02721   <span class="comment">// -- Report data:</span>
<a name="l02722"></a>02722   printf(<span class="stringliteral">&quot;\n\n          *** IMAGE TALLY PERFORMANCE REPORT ***\n&quot;</span>);
<a name="l02723"></a>02723   
<a name="l02724"></a>02724   <span class="keywordflow">if</span>(num_projections!=1)   <span class="comment">// Output the projection angle when simulating a CT:</span>
<a name="l02725"></a>02725   {
<a name="l02726"></a>02726     printf(<span class="stringliteral">&quot;              CT projection %d of %d: angle from X axis = %lf \n&quot;</span>, current_projection+1, num_projections, current_angle*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>);
<a name="l02727"></a>02727   }
<a name="l02728"></a>02728   
<a name="l02729"></a>02729   printf(<span class="stringliteral">&quot;              Simulated x rays:    %lld\n&quot;</span>, total_histories);
<a name="l02730"></a>02730   printf(<span class="stringliteral">&quot;              Simulation time [s]: %.2f\n&quot;</span>, time_elapsed);
<a name="l02731"></a>02731   <span class="keywordflow">if</span> (time_elapsed&gt;0.000001)
<a name="l02732"></a>02732     printf(<span class="stringliteral">&quot;              Speed [x-rays/s]:    %.2f\n\n&quot;</span>, ((<span class="keywordtype">double</span>)total_histories)/time_elapsed);
<a name="l02733"></a>02733 
<a name="l02734"></a>02734   FILE* file_ptr = fopen(file_name_output, <span class="stringliteral">&quot;w&quot;</span>);
<a name="l02735"></a>02735   
<a name="l02736"></a>02736   <span class="keywordflow">if</span> (file_ptr==NULL)
<a name="l02737"></a>02737   {
<a name="l02738"></a>02738     printf(<span class="stringliteral">&quot;\n\n   !!fopen ERROR report_image!! File %s can not be opened!!\n&quot;</span>, file_name_output);
<a name="l02739"></a>02739     exit(-3);
<a name="l02740"></a>02740   }
<a name="l02741"></a>02741   
<a name="l02742"></a>02742   fprintf(file_ptr, <span class="stringliteral">&quot;# \n&quot;</span>);
<a name="l02743"></a>02743   fprintf(file_ptr, <span class="stringliteral">&quot;#     *****************************************************************************\n&quot;</span>);
<a name="l02744"></a>02744   fprintf(file_ptr, <span class="stringliteral">&quot;#     ***         MC-GPU, version 1.3 (http://code.google.com/p/mcgpu/)         ***\n&quot;</span>);
<a name="l02745"></a>02745   fprintf(file_ptr, <span class="stringliteral">&quot;#     ***                                                                       ***\n&quot;</span>);
<a name="l02746"></a>02746   fprintf(file_ptr, <span class="stringliteral">&quot;#     ***                     Andreu Badal (Andreu.Badal-Soler@fda.hhs.gov)     ***\n&quot;</span>);
<a name="l02747"></a>02747   fprintf(file_ptr, <span class="stringliteral">&quot;#     *****************************************************************************\n&quot;</span>);
<a name="l02748"></a>02748   fprintf(file_ptr, <span class="stringliteral">&quot;# \n&quot;</span>);  
<a name="l02749"></a>02749 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l02750"></a>02750 <span class="preprocessor"></span>  fprintf(file_ptr, <span class="stringliteral">&quot;#  *** SIMULATION IN THE GPU USING CUDA ***\n&quot;</span>);
<a name="l02751"></a>02751 <span class="preprocessor">#else</span>
<a name="l02752"></a>02752 <span class="preprocessor"></span>  fprintf(file_ptr, <span class="stringliteral">&quot;#  *** SIMULATION IN THE CPU ***\n&quot;</span>);
<a name="l02753"></a>02753 <span class="preprocessor">#endif  </span>
<a name="l02754"></a>02754 <span class="preprocessor"></span>  fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);
<a name="l02755"></a>02755   fprintf(file_ptr, <span class="stringliteral">&quot;#  Image created counting the energy arriving at each pixel: ideal energy integrating detector.\n&quot;</span>);
<a name="l02756"></a>02756   fprintf(file_ptr, <span class="stringliteral">&quot;#  Pixel value units: eV/cm^2 per history (energy fluence).\n&quot;</span>);
<a name="l02757"></a>02757 
<a name="l02758"></a>02758 
<a name="l02759"></a>02759   <span class="keywordflow">if</span>(num_projections!=1)   <span class="comment">// Output the projection angle when simulating a CT:</span>
<a name="l02760"></a>02760   {
<a name="l02761"></a>02761     fprintf(file_ptr, <span class="stringliteral">&quot;#  CT projection %d of %d: angle from X axis = %lf \n&quot;</span>, current_projection+1, num_projections, current_angle*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>);
<a name="l02762"></a>02762   }  
<a name="l02763"></a>02763 
<a name="l02764"></a>02764   fprintf(file_ptr, <span class="stringliteral">&quot;#  Focal spot position = (%.8f,%.8f,%.8f), cone beam direction = (%.8f,%.8f,%.8f)\n&quot;</span>, source_data[current_projection].position.x, source_data[current_projection].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, source_data[current_projection].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>, source_data[current_projection].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>, source_data[current_projection].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>, source_data[current_projection].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>);
<a name="l02765"></a>02765 
<a name="l02766"></a>02766   fprintf(file_ptr, <span class="stringliteral">&quot;#  Pixel size:  %lf x %lf = %lf cm^2\n&quot;</span>, 1.0/(<span class="keywordtype">double</span>)(detector_data[0].<a class="code" href="structdetector__struct.html#a3c6f9e59e89db06211ccea63287e2826">inv_pixel_size_X</a>), 1.0/(<span class="keywordtype">double</span>)(detector_data[0].<a class="code" href="structdetector__struct.html#aa91fc94f674ad7af8eca577f4d8cd22b">inv_pixel_size_Z</a>), 1.0/(<span class="keywordtype">double</span>)(detector_data[0].inv_pixel_size_X*detector_data[0].inv_pixel_size_Z));
<a name="l02767"></a>02767   
<a name="l02768"></a>02768   fprintf(file_ptr, <span class="stringliteral">&quot;#  Number of pixels in X and Z:  %d  %d\n&quot;</span>, detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a>, detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>);
<a name="l02769"></a>02769   fprintf(file_ptr, <span class="stringliteral">&quot;#  (X rows given first, a blank line separates the different Z values)\n&quot;</span>);
<a name="l02770"></a>02770   fprintf(file_ptr, <span class="stringliteral">&quot;# \n&quot;</span>);
<a name="l02771"></a>02771   fprintf(file_ptr, <span class="stringliteral">&quot;#  [NON-SCATTERED] [COMPTON] [RAYLEIGH] [MULTIPLE-SCATTING]\n&quot;</span>);
<a name="l02772"></a>02772   fprintf(file_ptr, <span class="stringliteral">&quot;# ==========================================================\n&quot;</span>);
<a name="l02773"></a>02773 
<a name="l02774"></a>02774   <span class="keyword">const</span> <span class="keywordtype">double</span> SCALE = 1.0/<a class="code" href="MC-GPU__v1_83_8h.html#ac0c03cf5e803404f5dc2a42467395a3a" title="Value to scale the deposited energy in the pixels so that it can be stored as a long long integer ins...">SCALE_eV</a>;    <span class="comment">// conversion to eV using the inverse of the constant used in the &quot;tally_image&quot; kernel function (defined in the header file)</span>
<a name="l02775"></a>02775   <span class="keyword">const</span> <span class="keywordtype">double</span> NORM = SCALE * detector_data[0].<a class="code" href="structdetector__struct.html#a3c6f9e59e89db06211ccea63287e2826">inv_pixel_size_X</a> * detector_data[0].<a class="code" href="structdetector__struct.html#aa91fc94f674ad7af8eca577f4d8cd22b">inv_pixel_size_Z</a> / ((double)total_histories);  <span class="comment">// ==&gt; [eV/cm^2 per history]</span>
<a name="l02776"></a>02776   <span class="keywordtype">double</span> energy_noScatter, energy_compton, energy_rayleigh, energy_multiscatter;
<a name="l02777"></a>02777   <span class="keywordtype">double</span> energy_integral = 0.0;   <span class="comment">// Integrate (add) the energy in the image pixels [meV]</span>
<a name="l02778"></a>02778   <span class="keywordtype">double</span> maximum_energy_pixel = -100.0;  <span class="comment">// Find maximum pixel signal</span>
<a name="l02779"></a>02779   <span class="keywordtype">int</span> maximum_energy_pixel_x=0, maximum_energy_pixel_y=0, maximum_energy_pixel_number=0;   
<a name="l02780"></a>02780 
<a name="l02781"></a>02781   <span class="keywordtype">int</span> pixels_per_image = (detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a>*detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>), pixel=0;
<a name="l02782"></a>02782   <span class="keywordtype">int</span> i, j;
<a name="l02783"></a>02783   <span class="keywordflow">for</span>(j=0; j&lt;detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a3d7ff3bd7734f8c2f38613a4da5e6c83">y</a>; j++)
<a name="l02784"></a>02784   {
<a name="l02785"></a>02785     <span class="keywordflow">for</span>(i=0; i&lt;detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>.<a class="code" href="structint2.html#a2b4908d6f7ae421393c0ec0233e90551">x</a>; i++)
<a name="l02786"></a>02786     {
<a name="l02787"></a>02787       energy_noScatter    = (double)(image[pixel]);
<a name="l02788"></a>02788       energy_compton      = (double)(image[pixel +   pixels_per_image]);
<a name="l02789"></a>02789       energy_rayleigh     = (double)(image[pixel + 2*pixels_per_image]);
<a name="l02790"></a>02790       energy_multiscatter = (double)(image[pixel + 3*pixels_per_image]);
<a name="l02791"></a>02791 
<a name="l02792"></a>02792       <span class="comment">// -- Write the results in an external file; the image corresponding to all particles not written: it has to be infered adding all images</span>
<a name="l02793"></a>02793       fprintf(file_ptr, <span class="stringliteral">&quot;%.8lf %.8lf %.8lf %.8lf\n&quot;</span>, NORM*energy_noScatter, NORM*energy_compton, NORM*energy_rayleigh, NORM*energy_multiscatter);
<a name="l02794"></a>02794       
<a name="l02795"></a>02795       <span class="keyword">register</span> <span class="keywordtype">double</span> total_energy_pixel = energy_noScatter + energy_compton + energy_rayleigh + energy_multiscatter;   <span class="comment">// Find and report the pixel with maximum signal</span>
<a name="l02796"></a>02796       <span class="keywordflow">if</span> (total_energy_pixel&gt;maximum_energy_pixel)
<a name="l02797"></a>02797       {
<a name="l02798"></a>02798         maximum_energy_pixel = total_energy_pixel;
<a name="l02799"></a>02799         maximum_energy_pixel_x = i;
<a name="l02800"></a>02800         maximum_energy_pixel_y = j;
<a name="l02801"></a>02801         maximum_energy_pixel_number = pixel;
<a name="l02802"></a>02802       }            
<a name="l02803"></a>02803       energy_integral += total_energy_pixel;   <span class="comment">// Count total energy in the whole image      </span>
<a name="l02804"></a>02804       
<a name="l02805"></a>02805       pixel++;   <span class="comment">// Move to next pixel</span>
<a name="l02806"></a>02806     }
<a name="l02807"></a>02807     fprintf(file_ptr, <span class="stringliteral">&quot;\n&quot;</span>);     <span class="comment">// Separate rows with an empty line for visualization with gnuplot.</span>
<a name="l02808"></a>02808   }
<a name="l02809"></a>02809   
<a name="l02810"></a>02810   fprintf(file_ptr, <span class="stringliteral">&quot;#   *** Simulation REPORT: ***\n&quot;</span>);
<a name="l02811"></a>02811   fprintf(file_ptr, <span class="stringliteral">&quot;#       Fraction of energy detected (over the mean energy of the spectrum): %.3lf%%\n&quot;</span>, 100.0*SCALE*(energy_integral/(<span class="keywordtype">double</span>)(total_histories))/(<span class="keywordtype">double</span>)(mean_energy_spectrum));
<a name="l02812"></a>02812   fprintf(file_ptr, <span class="stringliteral">&quot;#       Maximum energy detected in pixel %i: (x,y)=(%i,%i) -&gt; pixel value = %lf eV/cm^2\n&quot;</span>, maximum_energy_pixel_number, maximum_energy_pixel_x, maximum_energy_pixel_y, NORM*maximum_energy_pixel);
<a name="l02813"></a>02813   fprintf(file_ptr, <span class="stringliteral">&quot;#       Simulated x rays:    %lld\n&quot;</span>, total_histories);
<a name="l02814"></a>02814   fprintf(file_ptr, <span class="stringliteral">&quot;#       Simulation time [s]: %.2f\n&quot;</span>, time_elapsed);
<a name="l02815"></a>02815   <span class="keywordflow">if</span> (time_elapsed&gt;0.000001)
<a name="l02816"></a>02816     fprintf(file_ptr, <span class="stringliteral">&quot;#       Speed [x-rays/sec]:  %.2f\n\n&quot;</span>, ((<span class="keywordtype">double</span>)total_histories)/time_elapsed);
<a name="l02817"></a>02817    
<a name="l02818"></a>02818   fclose(file_ptr);  <span class="comment">// Close output file and flush stream</span>
<a name="l02819"></a>02819 
<a name="l02820"></a>02820   printf(<span class="stringliteral">&quot;              Fraction of initial energy arriving at the detector (over the mean energy of the spectrum):  %.3lf%%\n&quot;</span>, 100.0*SCALE*(energy_integral/(<span class="keywordtype">double</span>)(total_histories))/(<span class="keywordtype">double</span>)(mean_energy_spectrum));
<a name="l02821"></a>02821   printf(<span class="stringliteral">&quot;              Maximum energy detected in pixel %i: (x,y)=(%i,%i). Maximum pixel value = %lf eV/cm^2\n\n&quot;</span>, maximum_energy_pixel_number, maximum_energy_pixel_x, maximum_energy_pixel_y, NORM*maximum_energy_pixel);  
<a name="l02822"></a>02822   fflush(stdout);
<a name="l02823"></a>02823   
<a name="l02824"></a>02824   
<a name="l02825"></a>02825   <span class="comment">// -- Binary output:   </span>
<a name="l02826"></a>02826   <span class="keywordtype">float</span> energy_float;
<a name="l02827"></a>02827   <span class="keywordtype">char</span> file_binary[250];
<a name="l02828"></a>02828   strncpy (file_binary, file_name_output, 250);
<a name="l02829"></a>02829   strcat(file_binary,<span class="stringliteral">&quot;.raw&quot;</span>);                       <span class="comment">// !!BINARY!! </span>
<a name="l02830"></a>02830   FILE* file_binary_ptr = fopen(file_binary, <span class="stringliteral">&quot;w&quot;</span>);  <span class="comment">// !!BINARY!!</span>
<a name="l02831"></a>02831   <span class="keywordflow">if</span> (file_binary_ptr==NULL)
<a name="l02832"></a>02832   {
<a name="l02833"></a>02833     printf(<span class="stringliteral">&quot;\n\n   !!fopen ERROR report_image!! Binary file %s can not be opened for writing!!\n&quot;</span>, file_binary);
<a name="l02834"></a>02834     exit(-3);
<a name="l02835"></a>02835   }
<a name="l02836"></a>02836   
<a name="l02837"></a>02837   <span class="keywordflow">for</span>(i=0; i&lt;pixels_per_image; i++)
<a name="l02838"></a>02838   {
<a name="l02839"></a>02839     energy_float = (float)( NORM * (<span class="keywordtype">double</span>)(image[i] + image[i + pixels_per_image] + image[i + 2*pixels_per_image] + image[i + 3*pixels_per_image]) );  <span class="comment">// Total image (scatter + primary)</span>
<a name="l02840"></a>02840     fwrite(&amp;energy_float, <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_ptr);   <span class="comment">// Write pixel data in a binary file that can be easyly open in imageJ. !!BINARY!!</span>
<a name="l02841"></a>02841   }
<a name="l02842"></a>02842   <span class="keywordflow">for</span>(i=0; i&lt;pixels_per_image; i++)
<a name="l02843"></a>02843   {
<a name="l02844"></a>02844     energy_float = (float)( NORM * (<span class="keywordtype">double</span>)(image[i]) );  <span class="comment">// Non-scattered image</span>
<a name="l02845"></a>02845     fwrite(&amp;energy_float, <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_ptr);
<a name="l02846"></a>02846   }
<a name="l02847"></a>02847   <span class="keywordflow">for</span>(i=0; i&lt;pixels_per_image; i++)
<a name="l02848"></a>02848   {
<a name="l02849"></a>02849     energy_float = (float)( NORM * (<span class="keywordtype">double</span>)(image[i + pixels_per_image]) );  <span class="comment">// Compton image</span>
<a name="l02850"></a>02850     fwrite(&amp;energy_float, <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_ptr);
<a name="l02851"></a>02851   }
<a name="l02852"></a>02852   <span class="keywordflow">for</span>(i=0; i&lt;pixels_per_image; i++)
<a name="l02853"></a>02853   {
<a name="l02854"></a>02854     energy_float = (float)( NORM * (<span class="keywordtype">double</span>)(image[i + 2*pixels_per_image]) );  <span class="comment">// Rayleigh image</span>
<a name="l02855"></a>02855     fwrite(&amp;energy_float, <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_ptr);
<a name="l02856"></a>02856   }
<a name="l02857"></a>02857   <span class="keywordflow">for</span>(i=0; i&lt;pixels_per_image; i++)
<a name="l02858"></a>02858   {
<a name="l02859"></a>02859     energy_float = (float)( NORM * (<span class="keywordtype">double</span>)(image[i + 3*pixels_per_image]) );  <span class="comment">// Multiple-scatter image</span>
<a name="l02860"></a>02860     fwrite(&amp;energy_float, <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_ptr);
<a name="l02861"></a>02861   }       
<a name="l02862"></a>02862   
<a name="l02863"></a>02863   fclose(file_binary_ptr);    
<a name="l02864"></a>02864   
<a name="l02865"></a>02865     
<a name="l02866"></a>02866   <span class="keywordflow">return</span> 0;     <span class="comment">// Report could return not 0 to continue the simulation...</span>
<a name="l02867"></a>02867 }<span class="comment"></span>
<a name="l02868"></a>02868 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l02869"></a>02869 <span class="comment"></span>
<a name="l02870"></a>02870 
<a name="l02871"></a>02871 
<a name="l02872"></a>02872 <span class="comment"></span>
<a name="l02873"></a>02873 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l02874"></a>02874 <span class="comment">//! Report the total tallied 3D voxel dose deposition for all projections.</span>
<a name="l02875"></a>02875 <span class="comment">//! The voxel doses in the input ROI and their respective uncertainties are reported </span>
<a name="l02876"></a>02876 <span class="comment">//! in binary form (32-bit floats) in two separate .raw files.</span>
<a name="l02877"></a>02877 <span class="comment">//! The dose in a single plane at the level of the focal spot is also reported in  </span>
<a name="l02878"></a>02878 <span class="comment">//! ASCII format for simple visualization with GNUPLOT.</span>
<a name="l02879"></a>02879 <span class="comment">//! The total dose deposited in each different material is reported to the standard output.</span>
<a name="l02880"></a>02880 <span class="comment">//! The material dose is calculated adding the energy deposited in the individual voxels </span>
<a name="l02881"></a>02881 <span class="comment">//! within the dose ROI, and dividing by the total mass of the material in the ROI.</span>
<a name="l02882"></a>02882 <span class="comment">//!</span>
<a name="l02883"></a>02883 <span class="comment">//!       @param[in] file_dose_output   File where tallied image is reported</span>
<a name="l02884"></a>02884 <span class="comment">//!       @param[in] detector_data   Detector description read from the input file (pointer to detector_struct)</span>
<a name="l02885"></a>02885 <span class="comment">//!       @param[in] image  Tallied image (in meV per pixel)</span>
<a name="l02886"></a>02886 <span class="comment">//!       @param[in] time_elapsed   Time elapsed during the main loop execution (in seconds)</span>
<a name="l02887"></a>02887 <span class="comment">//!       @param[in] total_histories   Total number of x-rays simulated</span>
<a name="l02888"></a>02888 <span class="comment">//!       @param[in] source_data   Data required to compute the voxel plane to report in ASCII format: Z at the level of the source, 1st projection</span>
<a name="l02889"></a>02889 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l02890"></a><a class="code" href="MC-GPU__v1_83_8h.html#ac91d2c9046a60b6d7f225c573c4d4d98">02890</a> <span class="comment"></span><span class="keywordtype">int</span> <a class="code" href="MC-GPU__v1_83_8cu.html#aeceeb305fc1d995c4730e0fcedf10d0c" title="Report the total tallied 3D voxel dose deposition for all projections.">report_voxels_dose</a>(<span class="keywordtype">char</span>* file_dose_output, <span class="keywordtype">int</span> num_projections, <span class="keyword">struct</span> <a class="code" href="structvoxel__struct.html" title="Structure defining a voxelized box with the back-lower corner at the coordinate origin.">voxel_struct</a>* voxel_data, <a class="code" href="structfloat2.html">float2</a>* voxel_mat_dens, <a class="code" href="structulonglong2.html">ulonglong2</a>* voxels_Edep, <span class="keywordtype">double</span> time_total_MC_init_report, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories, <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_x_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_x_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_y_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_y_max, <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_z_min, <span class="keywordtype">short</span> <span class="keywordtype">int</span> dose_ROI_z_max, <span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>* source_data)
<a name="l02891"></a>02891 {
<a name="l02892"></a>02892 
<a name="l02893"></a>02893   printf(<span class="stringliteral">&quot;\n\n          *** VOXEL ROI DOSE TALLY REPORT ***\n\n&quot;</span>);
<a name="l02894"></a>02894     
<a name="l02895"></a>02895   FILE* file_ptr = fopen(file_dose_output, <span class="stringliteral">&quot;w&quot;</span>);
<a name="l02896"></a>02896   <span class="keywordflow">if</span> (file_ptr==NULL)
<a name="l02897"></a>02897   {
<a name="l02898"></a>02898     printf(<span class="stringliteral">&quot;\n\n   !!fopen ERROR report_voxels_dose!! File %s can not be opened!!\n&quot;</span>, file_dose_output);
<a name="l02899"></a>02899     exit(-3);
<a name="l02900"></a>02900   }    
<a name="l02901"></a>02901     
<a name="l02902"></a>02902   <span class="comment">// -- Binary output:                                         // !!BINARY!!  </span>
<a name="l02903"></a>02903   <span class="keywordtype">char</span> file_binary_mean[250], file_binary_sigma[250];
<a name="l02904"></a>02904   strncpy (file_binary_mean, file_dose_output, 250);
<a name="l02905"></a>02905   strcat(file_binary_mean,<span class="stringliteral">&quot;.raw&quot;</span>);                     
<a name="l02906"></a>02906   strncpy (file_binary_sigma, file_dose_output, 250);
<a name="l02907"></a>02907   strcat(file_binary_sigma,<span class="stringliteral">&quot;_2sigma.raw&quot;</span>);    
<a name="l02908"></a>02908   FILE* file_binary_mean_ptr  = fopen(file_binary_mean, <span class="stringliteral">&quot;w&quot;</span>);  <span class="comment">// !!BINARY!!</span>
<a name="l02909"></a>02909   FILE* file_binary_sigma_ptr = fopen(file_binary_sigma, <span class="stringliteral">&quot;w&quot;</span>);       <span class="comment">// !!BINARY!!</span>
<a name="l02910"></a>02910   <span class="keywordflow">if</span> (file_binary_mean_ptr==NULL)
<a name="l02911"></a>02911   {
<a name="l02912"></a>02912     printf(<span class="stringliteral">&quot;\n\n   !!fopen ERROR report_voxels_dose!! Binary file %s can not be opened!!\n&quot;</span>, file_dose_output);
<a name="l02913"></a>02913     exit(-3);
<a name="l02914"></a>02914   }
<a name="l02915"></a>02915   
<a name="l02916"></a>02916   <span class="keywordtype">int</span> DX = dose_ROI_x_max - dose_ROI_x_min + 1,
<a name="l02917"></a>02917       DY = dose_ROI_y_max - dose_ROI_y_min + 1,
<a name="l02918"></a>02918       DZ = dose_ROI_z_max - dose_ROI_z_min + 1;           
<a name="l02919"></a>02919       
<a name="l02920"></a>02920   <span class="comment">// -- Calculate the dose plane that will be output as ASCII text:</span>
<a name="l02921"></a>02921   <span class="keywordtype">int</span> z_plane_dose = (int)(source_data[0].position.z * voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> + 0.00001f);  <span class="comment">// Select voxel plane at the level of the source, 1st projections</span>
<a name="l02922"></a>02922   <span class="keywordflow">if</span> ( (z_plane_dose&lt;dose_ROI_z_min) || (z_plane_dose&gt;dose_ROI_z_max) )
<a name="l02923"></a>02923     z_plane_dose = (dose_ROI_z_max+dose_ROI_z_min)/2;
<a name="l02924"></a>02924   
<a name="l02925"></a>02925   <span class="keywordtype">int</span> z_plane_dose_ROI = z_plane_dose - dose_ROI_z_min;
<a name="l02926"></a>02926 
<a name="l02927"></a>02927   printf(<span class="stringliteral">&quot;              Reporting the 3D voxel dose distribution as binary floats in the .raw file, and the 2D dose for Z plane %d as ASCII text.\n&quot;</span>, z_plane_dose);
<a name="l02928"></a>02928 <span class="comment">//   printf(&quot;              Also reporting the dose to each material inside the input ROI adding the energy deposited in each individual voxel\n&quot;);</span>
<a name="l02929"></a>02929 <span class="comment">//   printf(&quot;              (these material dose results will be equal to the materials dose tally below if the ROI covers all the voxels).\n&quot;);</span>
<a name="l02930"></a>02930   
<a name="l02931"></a>02931   fprintf(file_ptr, <span class="stringliteral">&quot;# \n&quot;</span>);
<a name="l02932"></a>02932   fprintf(file_ptr, <span class="stringliteral">&quot;#     *****************************************************************************\n&quot;</span>);
<a name="l02933"></a>02933   fprintf(file_ptr, <span class="stringliteral">&quot;#     ***         MC-GPU, version 1.3 (http://code.google.com/p/mcgpu/)         ***\n&quot;</span>);
<a name="l02934"></a>02934   fprintf(file_ptr, <span class="stringliteral">&quot;#     ***                                                                       ***\n&quot;</span>);
<a name="l02935"></a>02935   fprintf(file_ptr, <span class="stringliteral">&quot;#     ***                     Andreu Badal (Andreu.Badal-Soler@fda.hhs.gov)     ***\n&quot;</span>);
<a name="l02936"></a>02936   fprintf(file_ptr, <span class="stringliteral">&quot;#     *****************************************************************************\n&quot;</span>);
<a name="l02937"></a>02937   fprintf(file_ptr, <span class="stringliteral">&quot;# \n&quot;</span>);  
<a name="l02938"></a>02938 <span class="preprocessor">#ifdef USING_CUDA</span>
<a name="l02939"></a>02939 <span class="preprocessor"></span>  fprintf(file_ptr, <span class="stringliteral">&quot;#  *** SIMULATION IN THE GPU USING CUDA ***\n&quot;</span>);
<a name="l02940"></a>02940 <span class="preprocessor">#else</span>
<a name="l02941"></a>02941 <span class="preprocessor"></span>  fprintf(file_ptr, <span class="stringliteral">&quot;#  *** SIMULATION IN THE CPU ***\n&quot;</span>);
<a name="l02942"></a>02942 <span class="preprocessor">#endif</span>
<a name="l02943"></a>02943 <span class="preprocessor"></span>  fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);
<a name="l02944"></a>02944   
<a name="l02945"></a>02945   
<a name="l02946"></a>02946   <span class="comment">// Report only one dose plane in ASCII, all the other data in binary only:</span>
<a name="l02947"></a>02947 
<a name="l02948"></a>02948   fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);
<a name="l02949"></a>02949   fprintf(file_ptr, <span class="stringliteral">&quot;#  3D dose deposition map (and dose uncertainty) created tallying the energy deposited by photons inside each voxel of the input geometry.\n&quot;</span>);
<a name="l02950"></a>02950   fprintf(file_ptr, <span class="stringliteral">&quot;#  Electrons were not transported and therefore we are approximating that the dose is equal to the KERMA (energy released by the photons alone).\n&quot;</span>);
<a name="l02951"></a>02951   fprintf(file_ptr, <span class="stringliteral">&quot;#  This approximation is acceptable when there is electronic equilibrium and when the range of the secondary electrons is shorter than the voxel size.\n&quot;</span>);
<a name="l02952"></a>02952   fprintf(file_ptr, <span class="stringliteral">&quot;#  Usually the doses will be acceptable for photon energies below 1 MeV. The dose estimates may not be accurate at the interface of low density volumes.\n&quot;</span>);
<a name="l02953"></a>02953   fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);
<a name="l02954"></a>02954   fprintf(file_ptr, <span class="stringliteral">&quot;#  The 3D dose deposition is reported in binary form in the .raw files (data given as 32-bit floats). \n&quot;</span>);
<a name="l02955"></a>02955   fprintf(file_ptr, <span class="stringliteral">&quot;#  To reduce the memory use and the reporting time this text output reports only the 2D dose at the Z plane at the level\n&quot;</span>); 
<a name="l02956"></a>02956   fprintf(file_ptr, <span class="stringliteral">&quot;#  of the source focal spot: z_coord = %d (z_coord in ROI = %d)\n&quot;</span>, z_plane_dose, z_plane_dose_ROI);
<a name="l02957"></a>02957   fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);  
<a name="l02958"></a>02958   fprintf(file_ptr, <span class="stringliteral">&quot;#  The total dose deposited in each different material is reported to the standard output.\n&quot;</span>);
<a name="l02959"></a>02959   fprintf(file_ptr, <span class="stringliteral">&quot;#  The dose is calculated adding the energy deposited in the individual voxels within the dose ROI and dividing by the total mass of the material in the ROI.\n&quot;</span>);
<a name="l02960"></a>02960   fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);  
<a name="l02961"></a>02961   fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);
<a name="l02962"></a>02962   fprintf(file_ptr, <span class="stringliteral">&quot;#  Voxel size:  %lf x %lf x %lf = %lf cm^3\n&quot;</span>, 1.0/(<span class="keywordtype">double</span>)(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>), 1.0/(<span class="keywordtype">double</span>)(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>), 1.0/(<span class="keywordtype">double</span>)(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>), 1.0/(<span class="keywordtype">double</span>)(voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>));
<a name="l02963"></a>02963   fprintf(file_ptr, <span class="stringliteral">&quot;#  Number of voxels in the reported region of interest (ROI) X, Y and Z:\n&quot;</span>);
<a name="l02964"></a>02964   fprintf(file_ptr, <span class="stringliteral">&quot;#      %d  %d  %d\n&quot;</span>, DX, DY, DZ);
<a name="l02965"></a>02965   fprintf(file_ptr, <span class="stringliteral">&quot;#  Coordinates of the ROI inside the voxel volume = X[%d,%d], Y[%d,%d], Z[%d,%d]\n&quot;</span>, dose_ROI_x_min+1, dose_ROI_x_max+1, dose_ROI_y_min+1, dose_ROI_y_max+1, dose_ROI_z_min+1, dose_ROI_z_max+1);  <span class="comment">// Show ROI with index=1 for the first voxel instead of 0.</span>
<a name="l02966"></a>02966   fprintf(file_ptr, <span class="stringliteral">&quot;#\n&quot;</span>);
<a name="l02967"></a>02967   fprintf(file_ptr, <span class="stringliteral">&quot;#  Voxel dose units: eV/g per history\n&quot;</span>);
<a name="l02968"></a>02968   fprintf(file_ptr, <span class="stringliteral">&quot;#  X rows given first, then Y, then Z. One blank line separates the different Y, and two blanks the Z values (GNUPLOT format).\n&quot;</span>);
<a name="l02969"></a>02969   fprintf(file_ptr, <span class="stringliteral">&quot;#  The dose distribution is also reported with binary FLOAT values (.raw file) for easy visualization in ImageJ.\n&quot;</span>);
<a name="l02970"></a>02970   fprintf(file_ptr, <span class="stringliteral">&quot;# \n&quot;</span>);
<a name="l02971"></a>02971   fprintf(file_ptr, <span class="stringliteral">&quot;#    [DOSE]   [2*standard_deviation]\n&quot;</span>);
<a name="l02972"></a>02972   fprintf(file_ptr, <span class="stringliteral">&quot;# =====================================\n&quot;</span>);
<a name="l02973"></a>02973   fflush(file_ptr);
<a name="l02974"></a>02974   
<a name="l02975"></a>02975   <span class="keywordtype">double</span> voxel_dose, max_voxel_dose = -1.0, max_voxel_dose_std_dev = -1.0;
<a name="l02976"></a>02976   
<a name="l02977"></a>02977   <span class="keywordtype">int</span> max_dose_voxel_geometry=0, max_voxel_dose_x=-1, max_voxel_dose_y=-1, max_voxel_dose_z=-1;
<a name="l02978"></a>02978   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_energy_deposited = 0;
<a name="l02979"></a>02979   <span class="keywordtype">double</span> inv_SCALE_eV = 1.0 / <a class="code" href="MC-GPU__v1_83_8h.html#ac0c03cf5e803404f5dc2a42467395a3a" title="Value to scale the deposited energy in the pixels so that it can be stored as a long long integer ins...">SCALE_eV</a>,      <span class="comment">// conversion to eV using the inverse of the constant used in the tally function (defined in the header file).         </span>
<a name="l02980"></a>02980                 inv_N = 1.0 / (double)(total_histories*((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)num_projections));
<a name="l02981"></a>02981                                 
<a name="l02982"></a>02982   <span class="keyword">register</span> <span class="keywordtype">int</span> i, j, k, voxel=0;
<a name="l02983"></a>02983     
<a name="l02984"></a>02984   <span class="keywordtype">double</span> mat_Edep[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>], mat_Edep2[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>], mat_mass_ROI[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];    <span class="comment">// Arrays with the total energy, energy squared and mass of each material inside the ROI (mass and dose outside the ROI was not tallied).</span>
<a name="l02985"></a>02985   <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> mat_voxels[<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>];
<a name="l02986"></a>02986   <span class="keywordflow">for</span>(i=0; i&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; i++)
<a name="l02987"></a>02987   {
<a name="l02988"></a>02988      mat_Edep[i]  = 0.0;
<a name="l02989"></a>02989      mat_Edep2[i] = 0.0;
<a name="l02990"></a>02990      mat_mass_ROI[i]  = 0.0;
<a name="l02991"></a>02991      mat_voxels[i]= 0;
<a name="l02992"></a>02992   }
<a name="l02993"></a>02993   
<a name="l02994"></a>02994   <span class="keywordtype">double</span> voxel_volume = 1.0 / ( ((double)voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>) * ((double)voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>) * ((double)voxel_data-&gt;<a class="code" href="structvoxel__struct.html#ad1468b63b2b99455904cc43e9058d5c1">inv_voxel_size</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>) );
<a name="l02995"></a>02995     
<a name="l02996"></a>02996   <span class="keywordflow">for</span>(k=0; k&lt;DZ; k++)
<a name="l02997"></a>02997   {
<a name="l02998"></a>02998     <span class="keywordflow">for</span>(j=0; j&lt;DY; j++)
<a name="l02999"></a>02999     {
<a name="l03000"></a>03000       <span class="keywordflow">for</span>(i=0; i&lt;DX; i++)
<a name="l03001"></a>03001       {
<a name="l03002"></a>03002         <span class="keyword">register</span> <span class="keywordtype">int</span> voxel_geometry = (i+dose_ROI_x_min) + (j+dose_ROI_y_min)*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a> + (k+dose_ROI_z_min)*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a0a4ad50a155a35fa938ce6f16930affa">x</a>*voxel_data-&gt;<a class="code" href="structvoxel__struct.html#adeed35f7e44a7cfb2891001a1a68f868">num_voxels</a>.<a class="code" href="structint3.html#a5d95e23491677d61019f0354b16adca9">y</a>;
<a name="l03003"></a>03003         <span class="keyword">register</span> <span class="keywordtype">double</span> inv_voxel_mass = 1.0 / (voxel_mat_dens[voxel_geometry].<a class="code" href="structfloat2.html#a0f1298f22c4ee20a369cd3b9c25b2cc6">y</a>*voxel_volume);
<a name="l03004"></a>03004 
<a name="l03005"></a>03005         <span class="keyword">register</span> <span class="keywordtype">int</span> mat_number = (int)(voxel_mat_dens[voxel_geometry].x) - 1 ;  <span class="comment">// Material number, starting at 0.</span>
<a name="l03006"></a>03006         mat_mass_ROI[mat_number]  += voxel_mat_dens[voxel_geometry].<a class="code" href="structfloat2.html#a0f1298f22c4ee20a369cd3b9c25b2cc6">y</a>*voxel_volume;   <span class="comment">// Estimate mass and energy deposited in this material</span>
<a name="l03007"></a>03007         mat_Edep[mat_number]  += (double)voxels_Edep[voxel].x;        <span class="comment">// Using doubles to avoid overflow</span>
<a name="l03008"></a>03008         mat_Edep2[mat_number] += (double)voxels_Edep[voxel].y;
<a name="l03009"></a>03009         mat_voxels[mat_number]++;                                                <span class="comment">// Count voxels made of this material</span>
<a name="l03010"></a>03010         
<a name="l03011"></a>03011                 
<a name="l03012"></a>03012               <span class="comment">// Optional code to eliminate dose deposited in air (first material).  Sometimes useful for visualization (dose to air irrelevant, noisy)</span>
<a name="l03013"></a>03013               <span class="comment">//   if (voxel_mat_dens[voxel_geometry].x &lt; 1.1f)</span>
<a name="l03014"></a>03014               <span class="comment">//   {</span>
<a name="l03015"></a>03015               <span class="comment">//     voxels_Edep[voxel].x = 0.0f;</span>
<a name="l03016"></a>03016               <span class="comment">//     voxels_Edep[voxel].y = 0.0f;</span>
<a name="l03017"></a>03017               <span class="comment">//   }</span>
<a name="l03018"></a>03018                 
<a name="l03019"></a>03019         <span class="comment">// -- Convert total energy deposited to dose [eV/gram] per history:                        </span>
<a name="l03020"></a>03020         voxel_dose = ((double)voxels_Edep[voxel].x) * inv_N * inv_voxel_mass * inv_SCALE_eV;    <span class="comment">// [dose == Edep * voxel_volume / voxel_density / N_hist]                      </span>
<a name="l03021"></a>03021         total_energy_deposited += voxels_Edep[voxel].<a class="code" href="structulonglong2.html#a0d94e96b26765bf28a1a4b76391b88bb">x</a>;
<a name="l03022"></a>03022 
<a name="l03023"></a>03023         <span class="keyword">register</span> <span class="keywordtype">double</span> voxel_std_dev = (((double)voxels_Edep[voxel].y) * inv_N * inv_SCALE_eV * inv_voxel_mass - voxel_dose*voxel_dose) * inv_N;   <span class="comment">// [sigma = (&lt;Edep^2&gt; - &lt;Edep&gt;^2) / N_hist]</span>
<a name="l03024"></a>03024         <span class="keywordflow">if</span> (voxel_std_dev&gt;0.0)
<a name="l03025"></a>03025           voxel_std_dev = sqrt(voxel_std_dev);
<a name="l03026"></a>03026         
<a name="l03027"></a>03027         <span class="keywordflow">if</span> (voxel_dose &gt; max_voxel_dose)
<a name="l03028"></a>03028         {
<a name="l03029"></a>03029           <span class="comment">// Find the voxel that has the maximum dose:</span>
<a name="l03030"></a>03030           max_voxel_dose          = voxel_dose;
<a name="l03031"></a>03031           max_voxel_dose_std_dev  = voxel_std_dev;
<a name="l03032"></a>03032           max_voxel_dose_x        = i+dose_ROI_x_min;
<a name="l03033"></a>03033           max_voxel_dose_y        = j+dose_ROI_y_min;
<a name="l03034"></a>03034           max_voxel_dose_z        = k+dose_ROI_z_min;
<a name="l03035"></a>03035           max_dose_voxel_geometry = voxel_geometry;          
<a name="l03036"></a>03036         }
<a name="l03037"></a>03037         
<a name="l03038"></a>03038         <span class="comment">// Report only one dose plane in ASCII:</span>
<a name="l03039"></a>03039         <span class="keywordflow">if</span> (k == z_plane_dose_ROI) 
<a name="l03040"></a>03040           fprintf(file_ptr, <span class="stringliteral">&quot;%.6lf %.6lf\n&quot;</span>, voxel_dose, 2.0*voxel_std_dev);        
<a name="l03041"></a>03041         
<a name="l03042"></a>03042         <span class="keywordtype">float</span> voxel_dose_float  = (float)voxel_dose;         <span class="comment">// After dividing by the number of histories I can report FLOAT bc the number of significant digits will be low.  </span>
<a name="l03043"></a>03043         <span class="keywordtype">float</span> voxel_sigma_float = 2.0f * (float)(voxel_std_dev);
<a name="l03044"></a>03044         
<a name="l03045"></a>03045         fwrite(&amp;voxel_dose_float,  <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_mean_ptr);    <span class="comment">// Write dose data in a binary file that can be easyly open in imageJ.   !!BINARY!!</span>
<a name="l03046"></a>03046         fwrite(&amp;voxel_sigma_float, <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>), 1, file_binary_sigma_ptr);
<a name="l03047"></a>03047         
<a name="l03048"></a>03048         voxel++;
<a name="l03049"></a>03049       }
<a name="l03050"></a>03050       <span class="keywordflow">if</span> (k == z_plane_dose_ROI) 
<a name="l03051"></a>03051         fprintf(file_ptr, <span class="stringliteral">&quot;\n&quot;</span>);     <span class="comment">// Separate Ys with an empty line for visualization with gnuplot.</span>
<a name="l03052"></a>03052     }
<a name="l03053"></a>03053     <span class="keywordflow">if</span> (k == z_plane_dose_ROI) 
<a name="l03054"></a>03054       fprintf(file_ptr, <span class="stringliteral">&quot;\n&quot;</span>);     <span class="comment">// Separate Zs.</span>
<a name="l03055"></a>03055   }
<a name="l03056"></a>03056 
<a name="l03057"></a>03057   
<a name="l03058"></a>03058   fprintf(file_ptr, <span class="stringliteral">&quot;#   ****** DOSE REPORT: TOTAL SIMULATION PERFORMANCE FOR ALL PROJECTIONS ******\n&quot;</span>);
<a name="l03059"></a>03059   fprintf(file_ptr, <span class="stringliteral">&quot;#       Total number of simulated x rays: %lld\n&quot;</span>, total_histories*((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)num_projections));
<a name="l03060"></a>03060   fprintf(file_ptr, <span class="stringliteral">&quot;#       Simulated x rays per projection:  %lld\n&quot;</span>, total_histories);
<a name="l03061"></a>03061   fprintf(file_ptr, <span class="stringliteral">&quot;#       Total simulation time [s]:  %.2f\n&quot;</span>, time_total_MC_init_report);
<a name="l03062"></a>03062   <span class="keywordflow">if</span> (time_total_MC_init_report&gt;0.000001)
<a name="l03063"></a>03063     fprintf(file_ptr, <span class="stringliteral">&quot;#       Total speed [x-rays/s]:  %.2f\n&quot;</span>, (<span class="keywordtype">double</span>)(total_histories*((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)num_projections))/time_total_MC_init_report);
<a name="l03064"></a>03064 
<a name="l03065"></a>03065   
<a name="l03066"></a>03066   fprintf(file_ptr, <span class="stringliteral">&quot;\n#       Total energy absorved inside the dose ROI: %.5lf keV/hist\n\n&quot;</span>, 0.001*((<span class="keywordtype">double</span>)total_energy_deposited)*inv_N*inv_SCALE_eV);
<a name="l03067"></a>03067   
<a name="l03068"></a>03068   <span class="comment">// Output data to standard input:</span>
<a name="l03069"></a>03069   printf(<span class="stringliteral">&quot;\n              Total energy absorved inside the dose deposition ROI: %.5lf keV/hist\n&quot;</span>, 0.001*((<span class="keywordtype">double</span>)total_energy_deposited)*inv_N*inv_SCALE_eV);
<a name="l03070"></a>03070   <span class="keyword">register</span> <span class="keywordtype">double</span> voxel_mass_max_dose = voxel_volume*voxel_mat_dens[max_dose_voxel_geometry].<a class="code" href="structfloat2.html#a0f1298f22c4ee20a369cd3b9c25b2cc6">y</a>; 
<a name="l03071"></a>03071   printf(  <span class="stringliteral">&quot;              Maximum voxel dose (+-2 sigma): %lf +- %lf eV/g per history (E_dep_voxel=%lf eV/hist)\n&quot;</span>, max_voxel_dose, max_voxel_dose_std_dev, (max_voxel_dose*voxel_mass_max_dose));
<a name="l03072"></a>03072   printf(  <span class="stringliteral">&quot;              for the voxel: material=%d, density=%.8f g/cm^3, voxel_mass=%.8lf g, voxel coord in geometry=(%d,%d,%d)\n\n&quot;</span>, (<span class="keywordtype">int</span>)voxel_mat_dens[max_dose_voxel_geometry].x, voxel_mat_dens[max_dose_voxel_geometry].y, voxel_mass_max_dose, max_voxel_dose_x, max_voxel_dose_y, max_voxel_dose_z);
<a name="l03073"></a>03073   
<a name="l03074"></a>03074   
<a name="l03075"></a>03075   <span class="comment">// -- Report dose deposited in each material:  </span>
<a name="l03076"></a>03076   printf(<span class="stringliteral">&quot;              Dose deposited in the different materials inside the input ROI computed post-processing the 3D voxel dose results:\n\n&quot;</span>);
<a name="l03077"></a>03077   printf(<span class="stringliteral">&quot;    [MATERIAL]  [DOSE_ROI, eV/g/hist]  [2*std_dev]  [Rel error 2*std_dev, %%]  [E_dep [eV/hist]  [MASS_ROI, g]  [NUM_VOXELS_ROI]\n&quot;</span>);
<a name="l03078"></a>03078   printf(<span class="stringliteral">&quot;   =============================================================================================================================\n&quot;</span>);
<a name="l03079"></a>03079   
<a name="l03080"></a>03080   <span class="keywordflow">for</span>(i=0; i&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; i++)
<a name="l03081"></a>03081   {
<a name="l03082"></a>03082     <span class="keywordflow">if</span>(mat_voxels[i]&gt;0)   <span class="comment">// Report only for materials found at least in 1 voxel of the input geometry (prevent dividing by 0 mass).</span>
<a name="l03083"></a>03083     {
<a name="l03084"></a>03084       
<a name="l03085"></a>03085       <span class="keywordtype">double</span> Edep = mat_Edep[i] * inv_N * inv_SCALE_eV;    <span class="comment">// [dose == Edep/Mass/N_hist]</span>
<a name="l03086"></a>03086       <span class="comment">// !!DeBuG!! BUG in version 1.2: I have to divide by mass after computing the mean and sigma!!!</span>
<a name="l03087"></a>03087       <span class="comment">// !!DeBuG!! WRONG code:  double material_dose = mat_Edep[i] * inv_N  * inv_SCALE_eV / mat_mass_ROI[i];    // [dose == Edep/Mass/N_hist]</span>
<a name="l03088"></a>03088       <span class="comment">// !!DeBuG!! WRONG code:  double material_std_dev = (mat_Edep2[i] * inv_N  * inv_SCALE_eV / mat_mass_ROI[i] - material_dose*material_dose) * inv_N;   // [sigma^2 = (&lt;Edep^2&gt; - &lt;Edep&gt;^2) / N_hist]      </span>
<a name="l03089"></a>03089       
<a name="l03090"></a>03090       <span class="keywordtype">double</span> material_std_dev = (mat_Edep2[i] * inv_N - Edep*Edep) * inv_N;   <span class="comment">// [sigma^2 = (&lt;Edep^2&gt; - &lt;Edep&gt;^2) / N_hist]   (mat_Edep2 not scaled by SCALE_eV in kernel to prevent overflow)</span>
<a name="l03091"></a>03091       <span class="keywordflow">if</span> (material_std_dev&gt;0.0)
<a name="l03092"></a>03092         material_std_dev = sqrt(material_std_dev);
<a name="l03093"></a>03093       
<a name="l03094"></a>03094       <span class="keywordtype">double</span> material_dose = Edep / mat_mass_ROI[i];
<a name="l03095"></a>03095       material_std_dev = material_std_dev / mat_mass_ROI[i];
<a name="l03096"></a>03096       
<a name="l03097"></a>03097       <span class="keywordtype">double</span> rel_diff = 0.0;
<a name="l03098"></a>03098       <span class="keywordflow">if</span> (material_dose&gt;0.0)
<a name="l03099"></a>03099         rel_diff = material_std_dev/material_dose;
<a name="l03100"></a>03100     
<a name="l03101"></a>03101       printf(<span class="stringliteral">&quot;\t%d\t%.5lf\t\t%.5lf\t\t%.2lf\t\t%.2lf\t\t%.5lf\t%u\n&quot;</span>, (i+1), material_dose, 2.0*material_std_dev, (2.0*100.0*rel_diff), Edep, mat_mass_ROI[i], mat_voxels[i]);            
<a name="l03102"></a>03102  
<a name="l03103"></a>03103     }    
<a name="l03104"></a>03104   }       
<a name="l03105"></a>03105   printf(<span class="stringliteral">&quot;\n&quot;</span>);
<a name="l03106"></a>03106         
<a name="l03107"></a>03107   
<a name="l03108"></a>03108   fflush(stdout);          
<a name="l03109"></a>03109   fclose(file_ptr);  <span class="comment">// Close output file and flush stream</span>
<a name="l03110"></a>03110   fclose(file_binary_mean_ptr);
<a name="l03111"></a>03111   fclose(file_binary_sigma_ptr);
<a name="l03112"></a>03112 
<a name="l03113"></a>03113   <span class="keywordflow">return</span> 0;   <span class="comment">// Report could return not 0 to continue the simulation...</span>
<a name="l03114"></a>03114 }<span class="comment"></span>
<a name="l03115"></a>03115 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03116"></a>03116 <span class="comment"></span>
<a name="l03117"></a>03117 
<a name="l03118"></a>03118 <span class="comment"></span>
<a name="l03119"></a>03119 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03120"></a>03120 <span class="comment">//! Report the tallied dose to each material number, accounting for different </span>
<a name="l03121"></a>03121 <span class="comment">//! densities in different regions with the same material number. </span>
<a name="l03122"></a>03122 <span class="comment">//!</span>
<a name="l03123"></a>03123 <span class="comment">//!       @param[in] num_projections   Number of projections simulated</span>
<a name="l03124"></a>03124 <span class="comment">//!       @param[in] total_histories   Total number of x-rays simulated per projection</span>
<a name="l03125"></a>03125 <span class="comment">//!       @param[out] density_nominal   Array with the nominal densities of materials given in the input file; -1 for materials not defined. Used to report only defined materials.</span>
<a name="l03126"></a>03126 <span class="comment">//!       @param[in] materials_dose   Tallied dose and dose^2 arrays</span>
<a name="l03127"></a>03127 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l03128"></a><a class="code" href="MC-GPU__v1_83_8h.html#aa02ffc836f2b6f9de5386cadb11e30b7">03128</a> <span class="comment"></span><span class="keywordtype">int</span> <a class="code" href="MC-GPU__v1_83_8cu.html#aa02ffc836f2b6f9de5386cadb11e30b7" title="Report the tallied dose to each material number, accounting for different densities in different regi...">report_materials_dose</a>(<span class="keywordtype">int</span> num_projections, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories, <span class="keywordtype">float</span> *density_nominal, <a class="code" href="structulonglong2.html">ulonglong2</a> *materials_dose, <span class="keywordtype">double</span> *mass_materials)  <span class="comment">// !!tally_materials_dose!!</span>
<a name="l03129"></a>03129 {
<a name="l03130"></a>03130 
<a name="l03131"></a>03131   printf(<span class="stringliteral">&quot;\n\n          *** MATERIALS TOTAL DOSE TALLY REPORT ***\n\n&quot;</span>);  
<a name="l03132"></a>03132   printf(<span class="stringliteral">&quot;              Dose deposited in each material defined in the input file (tallied directly per material, not per voxel):\n&quot;</span>);
<a name="l03133"></a>03133   printf(<span class="stringliteral">&quot;              The results of this tally should be equal to the voxel tally doses for an ROI covering all voxels.\n\n&quot;</span>);
<a name="l03134"></a>03134   printf(<span class="stringliteral">&quot;    [MAT]  [DOSE, eV/g/hist]  [2*std_dev]  [Rel_error 2*std_dev, %%]  [E_dep [eV/hist]  [MASS_TOTAL, g]\n&quot;</span>);
<a name="l03135"></a>03135   printf(<span class="stringliteral">&quot;   ====================================================================================================\n&quot;</span>);
<a name="l03136"></a>03136   
<a name="l03137"></a>03137   <span class="keywordtype">double</span> dose, Edep, std_dev, rel_diff, inv_N = 1.0 / (double)(total_histories*((<span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span>)num_projections));
<a name="l03138"></a>03138   <span class="keywordtype">int</span> i, flag=0, max_mat=0;
<a name="l03139"></a>03139   <span class="keywordflow">for</span>(i=0; i&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a7aa308231ff3dcfd43ac19023e2e6e15" title="Constants values for the Compton and Rayleigh models:">MAX_MATERIALS</a>; i++)
<a name="l03140"></a>03140   {
<a name="l03141"></a>03141     <span class="keywordflow">if</span> (density_nominal[i]&lt;0.0f)
<a name="l03142"></a>03142       <span class="keywordflow">break</span>;  <span class="comment">// Skip report for materials not defined in the input file</span>
<a name="l03143"></a>03143     
<a name="l03144"></a>03144     Edep    = ((double)materials_dose[i].x) / <a class="code" href="MC-GPU__v1_83_8h.html#ac0c03cf5e803404f5dc2a42467395a3a" title="Value to scale the deposited energy in the pixels so that it can be stored as a long long integer ins...">SCALE_eV</a> * inv_N;
<a name="l03145"></a>03145     
<a name="l03146"></a>03146     std_dev = sqrt( (((<span class="keywordtype">double</span>)materials_dose[i].y)*inv_N - Edep*Edep) * inv_N );   <span class="comment">// [sigma^2 = (&lt;Edep^2&gt; - &lt;Edep&gt;^2) / N_hist]   (not scaling &quot;materials_dose[i].y&quot; by SCALE_eV in kernel to prevent overflow).</span>
<a name="l03147"></a>03147     
<a name="l03148"></a>03148     <span class="keywordflow">if</span> (Edep&gt;0.0)
<a name="l03149"></a>03149       rel_diff = std_dev/Edep;
<a name="l03150"></a>03150     <span class="keywordflow">else</span>
<a name="l03151"></a>03151       rel_diff = 0.0;
<a name="l03152"></a>03152 
<a name="l03153"></a>03153     dose    = Edep / mass_materials[i];
<a name="l03154"></a>03154     std_dev = std_dev / mass_materials[i];
<a name="l03155"></a>03155     
<a name="l03156"></a>03156     printf(<span class="stringliteral">&quot;\t%d\t%.5lf\t\t%.5lf\t\t%.2lf\t\t%.2lf\t\t%.5lf\n&quot;</span>, (i+1), dose, 2.0*std_dev, 2.0*100.0*rel_diff, Edep, mass_materials[i]);
<a name="l03157"></a>03157     
<a name="l03158"></a>03158        <span class="comment">// printf(&quot;\t%d\t%.5lf\t\t%.5lf\t\t%.2lf\t\t%.5lf\t\t%.5lf\t\t\t%llu,\t\t%llu\n&quot;, (i+1), dose, 2.0*std_dev, 2.0*100.0*rel_diff, Edep, mass_materials[i],  materials_dose[i].x, materials_dose[i].y);  //!!DeBuG!! VERBOSE output: counters not divided by num histories</span>
<a name="l03159"></a>03159     
<a name="l03160"></a>03160     
<a name="l03161"></a>03161     <span class="keywordflow">if</span> (materials_dose[i].x&gt;1e16 || dose!=abs(dose) || std_dev!=abs(std_dev))  <span class="comment">// !!DeBuG!!  Try to detect a possible overflow in any material: large counter or negative, nan value</span>
<a name="l03162"></a>03162     {
<a name="l03163"></a>03163       flag = 1;
<a name="l03164"></a>03164       <span class="keywordflow">if</span> (materials_dose[i].x&gt;materials_dose[max_mat].x) 
<a name="l03165"></a>03165         max_mat = i;
<a name="l03166"></a>03166     }
<a name="l03167"></a>03167   }
<a name="l03168"></a>03168   
<a name="l03169"></a>03169   <span class="keywordflow">if</span> (flag!=0)    <span class="comment">// !!DeBuG!! Try to detect a possible overflow: large counter or negative, nan value. The value of SCALE_eV can be reduced to prevent this overflow in some cases.</span>
<a name="l03170"></a>03170   {
<a name="l03171"></a>03171     printf(<span class="stringliteral">&quot;\n     WARNING: it is possible that the unsigned long long int counter used to tally the standard deviation overflowed (&gt;2^64).\n&quot;</span>);  <span class="comment">// !!DeBuG!! </span>
<a name="l03172"></a>03172     printf(<span class="stringliteral">&quot;              The standard deviation may be incorrectly measured, but it will surely be very small (&lt;&lt; 1%%).\n&quot;</span>);
<a name="l03173"></a>03173     printf(<span class="stringliteral">&quot;              Max counter (mat=%d): E_dep = %llu , E_dep^2 = %llu\n\n&quot;</span>, max_mat+1, materials_dose[max_mat].x, materials_dose[max_mat].y);
<a name="l03174"></a>03174   }
<a name="l03175"></a>03175   fflush(stdout);  
<a name="l03176"></a>03176   <span class="keywordflow">return</span> 0;
<a name="l03177"></a>03177 }
<a name="l03178"></a>03178 
<a name="l03179"></a>03179 <span class="comment"></span>
<a name="l03180"></a>03180 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03181"></a>03181 <span class="comment"></span>
<a name="l03182"></a>03182 
<a name="l03183"></a>03183 <span class="comment"></span>
<a name="l03184"></a>03184 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03185"></a>03185 <span class="comment">//!  Sets the CT trajectory: store in memory the source and detector rotations</span>
<a name="l03186"></a>03186 <span class="comment">//!  that are needed to calculate the multiple projections.</span>
<a name="l03187"></a>03187 <span class="comment">//!  The first projection (0) was previously initialized in function &quot;read_input&quot;.</span>
<a name="l03188"></a>03188 <span class="comment">//!  </span>
<a name="l03189"></a>03189 <span class="comment">//!</span>
<a name="l03190"></a>03190 <span class="comment">//!  ASSUMPTIONS: the CT scan plane must be perpendicular to the Z axis, ie,</span>
<a name="l03191"></a>03191 <span class="comment">//!               the initial direction of the particles must have w=0!</span>
<a name="l03192"></a>03192 <span class="comment">//!</span>
<a name="l03193"></a>03193 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03194"></a><a class="code" href="MC-GPU__v1_83_8h.html#a5f62e6c0430761bba20f1e546d396a27">03194</a> <span class="comment"></span><span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a5f62e6c0430761bba20f1e546d396a27" title="Sets the CT trajectory: store in memory the source and detector rotations that are needed to calculat...">set_CT_trajectory</a>(<span class="keywordtype">int</span> myID, <span class="keywordtype">int</span> num_projections, <span class="keywordtype">double</span> D_angle, <span class="keywordtype">double</span> angularROI_0, <span class="keywordtype">double</span> angularROI_1, <span class="keywordtype">double</span> SRotAxisD, <span class="keyword">struct</span> <a class="code" href="structsource__struct.html" title="Structure storing the data defining the source model (except for the energy spectrum).">source_struct</a>* source_data, <span class="keyword">struct</span> <a class="code" href="structdetector__struct.html" title="Structure storing the data defining the x-ray detector.">detector_struct</a>* detector_data, <span class="keywordtype">double</span> vertical_translation_per_projection)
<a name="l03195"></a>03195 {
<a name="l03196"></a>03196   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;\n    -- Setting the sources and detectors for the %d CT projections (MAX_NUM_PROJECTIONS=%d):\n&quot;</span>, num_projections, <a class="code" href="MC-GPU__v1_83_8h.html#a30b26edf408e1ff6034eb712ab7e173e" title="Maximum number of projections allowed in the CT simulation (not limited by the constant memory becaus...">MAX_NUM_PROJECTIONS</a>);
<a name="l03197"></a>03197   <span class="keywordtype">double</span> cos_rX, cos_rZ, sin_rX, sin_rZ, current_angle;
<a name="l03198"></a>03198 
<a name="l03199"></a>03199   <span class="comment">// --Set center of rotation at the input distance between source and detector:</span>
<a name="l03200"></a>03200   <a class="code" href="structfloat3.html">float3</a> center_rotation;
<a name="l03201"></a>03201   center_rotation.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> =  source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> * SRotAxisD;
<a name="l03202"></a>03202   center_rotation.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> =  source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> + source_data[0].<a class="code" href="structsource__struct.html#ae1a18388dff97d101b36aeaba9bd363a">direction</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> * SRotAxisD;
<a name="l03203"></a>03203   center_rotation.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> =  source_data[0].<a class="code" href="structsource__struct.html#af0d7a559e56da9df6b01dda2ac95a787">position</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>;      <span class="comment">//  + source_data[0].direction.z * SRotAxisD;   // w=0 all the time!!</span>
<a name="l03204"></a>03204 
<a name="l03205"></a>03205   <span class="comment">// --Angular span between projections:</span>
<a name="l03206"></a>03206 
<a name="l03207"></a>03207   <span class="comment">//  -Set initial angle for the source (180 degress less than the detector pointed by the direction vector; the zero angle is the X axis, increasing to +Y axis).</span>
<a name="l03208"></a>03208   current_angle = acos((<span class="keywordtype">double</span>)source_data[0].direction.x);
<a name="l03209"></a>03209   <span class="keywordflow">if</span> (source_data[0].direction.y&lt;0)
<a name="l03210"></a>03210     current_angle = -current_angle;     <span class="comment">// Correct for the fact that positive and negative angles have the same ACOS</span>
<a name="l03211"></a>03211   <span class="keywordflow">if</span> (current_angle&lt;0.0)
<a name="l03212"></a>03212     current_angle += 2.0*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Make sure the angle is not negative, between [0,360) degrees.</span>
<a name="l03213"></a>03213   current_angle = current_angle - <a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Correct the fact that the source is opposite to the detector (180 degrees difference).</span>
<a name="l03214"></a>03214   <span class="keywordflow">if</span> (current_angle&lt;0.0)
<a name="l03215"></a>03215     current_angle += 2.0*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Make sure the angle is not negative, between [0,360) degrees..</span>
<a name="l03216"></a>03216 
<a name="l03217"></a>03217   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;         &lt;&lt; Projection #1 &gt;&gt; initial_angle=%.8f , D_angle=%.8f\n&quot;</span>, current_angle*<a class="code" href="MC-GPU__v1_83_8h.html#ac5a945020d3528355cda82d383676736">RAD2DEG</a>, D_angle*RAD2DEG);
<a name="l03218"></a>03218   <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                             Source direction=(%.8f,%.8f,%.8f), position=(%.8f,%.8f,%.8f)\n&quot;</span>, source_data[0].direction.x,source_data[0].direction.y,source_data[0].direction.z, source_data[0].position.x,source_data[0].position.y,source_data[0].position.z);
<a name="l03219"></a>03219 
<a name="l03220"></a>03220   <span class="keywordtype">int</span> i;
<a name="l03221"></a>03221   <span class="keywordflow">for</span> (i=1; i&lt;num_projections; i++)   <span class="comment">// The first projection (i=0) was initialized in function &quot;read_input&quot;.</span>
<a name="l03222"></a>03222   {
<a name="l03223"></a>03223     <span class="comment">// --Init constant parameters to the values in projection 0:</span>
<a name="l03224"></a>03224     source_data[i].cos_theta_low = source_data[0].cos_theta_low;
<a name="l03225"></a>03225     source_data[i].phi_low = source_data[0].phi_low;
<a name="l03226"></a>03226     source_data[i].D_cos_theta = source_data[0].D_cos_theta;
<a name="l03227"></a>03227     source_data[i].D_phi = source_data[0].D_phi;
<a name="l03228"></a>03228     source_data[i].max_height_at_y1cm = source_data[0].max_height_at_y1cm;    
<a name="l03229"></a>03229     detector_data[i].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l03230"></a>03230     detector_data[i].<a class="code" href="structdetector__struct.html#a1f67c512205953e75552865ae7a2490a">width_X</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a1f67c512205953e75552865ae7a2490a">width_X</a>;
<a name="l03231"></a>03231     detector_data[i].<a class="code" href="structdetector__struct.html#af61592c77c6da548fff31215bf956dd8">height_Z</a> = detector_data[0].<a class="code" href="structdetector__struct.html#af61592c77c6da548fff31215bf956dd8">height_Z</a>;
<a name="l03232"></a>03232     detector_data[i].<a class="code" href="structdetector__struct.html#a3c6f9e59e89db06211ccea63287e2826">inv_pixel_size_X</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a3c6f9e59e89db06211ccea63287e2826">inv_pixel_size_X</a>;
<a name="l03233"></a>03233     detector_data[i].<a class="code" href="structdetector__struct.html#aa91fc94f674ad7af8eca577f4d8cd22b">inv_pixel_size_Z</a> = detector_data[0].<a class="code" href="structdetector__struct.html#aa91fc94f674ad7af8eca577f4d8cd22b">inv_pixel_size_Z</a>;
<a name="l03234"></a>03234     detector_data[i].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a> = detector_data[0].<a class="code" href="structdetector__struct.html#a1a30e7168cba7eebe918ab39bb408dce">num_pixels</a>;
<a name="l03235"></a>03235     detector_data[i].<a class="code" href="structdetector__struct.html#ac26d0b66ebf521346d8ba07609340ea6">total_num_pixels</a> = detector_data[0].<a class="code" href="structdetector__struct.html#ac26d0b66ebf521346d8ba07609340ea6">total_num_pixels</a>;
<a name="l03236"></a>03236     detector_data[i].<a class="code" href="structdetector__struct.html#adbe41a01886ae097dac0fa908551d035">rotation_flag</a> = detector_data[0].<a class="code" href="structdetector__struct.html#adbe41a01886ae097dac0fa908551d035">rotation_flag</a>;
<a name="l03237"></a>03237         
<a name="l03238"></a>03238         
<a name="l03239"></a>03239     <span class="comment">// --Set the new source location and direction, for the current CT projection:</span>
<a name="l03240"></a>03240     current_angle += D_angle;
<a name="l03241"></a>03241     <span class="keywordflow">if</span> (current_angle&gt;=(2.0*PI-0.0001))
<a name="l03242"></a>03242       current_angle -= 2.0*<a class="code" href="MC-GPU__v1_83_8h.html#a598a3330b3c21701223ee0ca14316eca">PI</a>;   <span class="comment">// Make sure the angle is not above or equal to 360 degrees.</span>
<a name="l03243"></a>03243 
<a name="l03244"></a>03244     source_data[i].position.x = center_rotation.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> + SRotAxisD*cos(current_angle);
<a name="l03245"></a>03245     source_data[i].position.y = center_rotation.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> + SRotAxisD*sin(current_angle);
<a name="l03246"></a>03246     source_data[i].position.z = source_data[i-1].position.z + vertical_translation_per_projection;   <span class="comment">//  The Z position can increase between projections for a helical scan. But rotation still around Z always: (w=0)!!</span>
<a name="l03247"></a>03247 
<a name="l03248"></a>03248     source_data[i].direction.x = center_rotation.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> - source_data[i].position.x;
<a name="l03249"></a>03249     source_data[i].direction.y = center_rotation.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> - source_data[i].position.y;
<a name="l03250"></a>03250     source_data[i].direction.z = 0.0f;    <span class="comment">//  center_rotation.z - source_data[0].position.z;   !! w=0 all the time!!  </span>
<a name="l03251"></a>03251 
<a name="l03252"></a>03252     <span class="keywordtype">double</span> norm = 1.0/sqrt((<span class="keywordtype">double</span>)source_data[i].direction.x*(<span class="keywordtype">double</span>)source_data[i].direction.x + (<span class="keywordtype">double</span>)source_data[i].direction.y*(<span class="keywordtype">double</span>)source_data[i].direction.y <span class="comment">/* + source_data[i].direction.z*source_data[i].direction.z*/</span>);
<a name="l03253"></a>03253     source_data[i].direction.x = (float)(((<span class="keywordtype">double</span>)source_data[i].direction.x)*norm);
<a name="l03254"></a>03254     source_data[i].direction.y = (float)(((<span class="keywordtype">double</span>)source_data[i].direction.y)*norm);
<a name="l03255"></a>03255       <span class="comment">// source_data[i].direction.z = (float)(((double)source_data[i].direction.z)*norm);</span>
<a name="l03256"></a>03256 
<a name="l03257"></a>03257     <span class="comment">// --Set the new detector in front of the new source:</span>
<a name="l03258"></a>03258     detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = source_data[i].position.x + source_data[i].direction.x * detector_data[i].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;   <span class="comment">// Set the center of the detector straight ahead of the focal spot.</span>
<a name="l03259"></a>03259     detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = source_data[i].position.y + source_data[i].direction.y * detector_data[i].<a class="code" href="structdetector__struct.html#a781104d94879829947e6c863ee17abb5">sdd</a>;
<a name="l03260"></a>03260     detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = source_data[i].position.z;    <span class="comment">//  + source_data[i].direction.z * detector_data[i].sdd;   !! w=0 all the time!!</span>
<a name="l03261"></a>03261 
<a name="l03262"></a>03262     <span class="keywordtype">double</span> rotX, rotZ;
<a name="l03263"></a>03263 
<a name="l03264"></a>03264       <span class="comment">//  detector_data[0].rotation_flag = 1;   //  Already set in read_input!</span>
<a name="l03265"></a>03265 
<a name="l03266"></a>03266     <span class="comment">// -- Rotate the detector center to +Y:</span>
<a name="l03267"></a>03267     <span class="comment">//    Set the rotation that will bring particles from the detector plane to +Y=(0,+1,0) through a rotation around X and around Z (counter-clock):</span>
<a name="l03268"></a>03268     rotX = 0.0;   <span class="comment">// !! w=0 all the time!!  CORRECT CALCULATION:  acos(source_data[0].direction.z) - 0.5*PI;  // Rotate to +Y = (0,+1,0) --&gt; rotX_0 =  -PI/2</span>
<a name="l03269"></a>03269 
<a name="l03270"></a>03270     <span class="keywordflow">if</span> ( (source_data[i].direction.x*source_data[i].direction.x + source_data[i].direction.y*source_data[i].direction.y) &gt; 1.0e-8 )   <span class="comment">// == u^2+v^2 &gt; 0</span>
<a name="l03271"></a>03271       <span class="keywordflow">if</span> (source_data[i].direction.y &gt;= 0.0f)
<a name="l03272"></a>03272         rotZ = 0.5*PI - acos(source_data[i].direction.x/sqrt(source_data[i].direction.x*source_data[i].direction.x + source_data[i].direction.y*source_data[i].direction.y));
<a name="l03273"></a>03273       <span class="keywordflow">else</span>
<a name="l03274"></a>03274         rotZ = 0.5*PI - (-acos(source_data[i].direction.x/sqrt(source_data[i].direction.x*source_data[i].direction.x + source_data[i].direction.y*source_data[i].direction.y)));
<a name="l03275"></a>03275     <span class="keywordflow">else</span>
<a name="l03276"></a>03276       rotZ = 0.0;   <span class="comment">// Vector pointing to +Z, do not rotate around Z then.</span>
<a name="l03277"></a>03277 
<a name="l03278"></a>03278     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;         &lt;&lt; Projection #%d &gt;&gt; current_angle=%.8f degrees (rotation around Z axis = %.8f)\n&quot;</span>, (i+1), current_angle*RAD2DEG, rotZ*RAD2DEG);
<a name="l03279"></a>03279     <a class="code" href="MC-GPU__v1_83_8h.html#a155f8b8dae8b676fc50d93a287c4e6df" title="MPI macro: mark commands to be executed only by the master thread (myID==0).">MASTER_THREAD</a> printf(<span class="stringliteral">&quot;                             Source direction = (%.8f,%.8f,%.8f) , position = (%.8f,%.8f,%.8f)\n&quot;</span>, source_data[i].direction.x,source_data[i].direction.y,source_data[i].direction.z, source_data[i].position.x,source_data[i].position.y,source_data[i].position.z);
<a name="l03280"></a>03280 
<a name="l03281"></a>03281     cos_rX = cos(rotX);
<a name="l03282"></a>03282     cos_rZ = cos(rotZ);
<a name="l03283"></a>03283     sin_rX = sin(rotX);
<a name="l03284"></a>03284     sin_rZ = sin(rotZ);
<a name="l03285"></a>03285     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[0] =  cos_rZ;    <span class="comment">// Rotation matrix RxRz:</span>
<a name="l03286"></a>03286     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[1] = -sin_rZ;
<a name="l03287"></a>03287     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[2] =  0.0f;
<a name="l03288"></a>03288     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[3] =  cos_rX*sin_rZ;
<a name="l03289"></a>03289     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[4] =  cos_rX*cos_rZ;
<a name="l03290"></a>03290     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[5] = -sin_rX;
<a name="l03291"></a>03291     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[6] =  sin_rX*sin_rZ;
<a name="l03292"></a>03292     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[7] =  sin_rX*cos_rZ;
<a name="l03293"></a>03293     detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[8] =  cos_rX;
<a name="l03294"></a>03294 
<a name="l03295"></a>03295 
<a name="l03296"></a>03296     detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[0] + detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[1] + detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[2];
<a name="l03297"></a>03297     detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a> = detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[3] + detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[4] + detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[5];
<a name="l03298"></a>03298     detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[6] + detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#aa6147d421a81889971f8c66aa92abf0d">y</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[7] + detector_data[i].<a class="code" href="structdetector__struct.html#a1f0ce65af62c45925dfc96de3b796474">center</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a>*detector_data[i].<a class="code" href="structdetector__struct.html#a05b03d920370a5925ea56932e5953a69">rot_inv</a>[8];
<a name="l03299"></a>03299 
<a name="l03300"></a>03300     <span class="comment">// -- Set the lower corner (minimum) coordinates at the normalized orientation: +Y. The detector has thickness 0.</span>
<a name="l03301"></a>03301     detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> = detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#af621f02abb1c788738fe61ea9807ff9c">x</a> - 0.5*detector_data[i].<a class="code" href="structdetector__struct.html#a1f67c512205953e75552865ae7a2490a">width_X</a>;
<a name="l03302"></a>03302 <span class="comment">//  detector_data[i].corner_min_rotated_to_Y.y = detector_data[i].corner_min_rotated_to_Y.y;</span>
<a name="l03303"></a>03303     detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> = detector_data[i].<a class="code" href="structdetector__struct.html#a824e406500b0d8ca59ca96985a599b8b">corner_min_rotated_to_Y</a>.<a class="code" href="structfloat3.html#a772dffd42d89f350c5a1b766c4703245">z</a> - 0.5*detector_data[i].<a class="code" href="structdetector__struct.html#af61592c77c6da548fff31215bf956dd8">height_Z</a>;
<a name="l03304"></a>03304 
<a name="l03305"></a>03305     <span class="comment">// *** Init the fan beam source model:</span>
<a name="l03306"></a>03306 
<a name="l03307"></a>03307       rotZ = -rotZ;   <span class="comment">// The source rotation is the inverse of the detector.</span>
<a name="l03308"></a>03308       cos_rX = cos(rotX);
<a name="l03309"></a>03309       cos_rZ = cos(rotZ);
<a name="l03310"></a>03310       sin_rX = sin(rotX);
<a name="l03311"></a>03311       sin_rZ = sin(rotZ);
<a name="l03312"></a>03312       <span class="comment">// --Rotation around X (alpha) and then around Z (phi): Rz*Rx (oposite of detector rotation)</span>
<a name="l03313"></a>03313       source_data[i].rot_fan[0] =  cos_rZ;
<a name="l03314"></a>03314       source_data[i].rot_fan[1] = -cos_rX*sin_rZ;
<a name="l03315"></a>03315       source_data[i].rot_fan[2] =  sin_rX*sin_rZ;
<a name="l03316"></a>03316       source_data[i].rot_fan[3] =  sin_rZ;
<a name="l03317"></a>03317       source_data[i].rot_fan[4] =  cos_rX*cos_rZ;
<a name="l03318"></a>03318       source_data[i].rot_fan[5] = -sin_rX*cos_rZ;
<a name="l03319"></a>03319       source_data[i].rot_fan[6] =  0.0f;
<a name="l03320"></a>03320       source_data[i].rot_fan[7] =  sin_rX;
<a name="l03321"></a>03321       source_data[i].rot_fan[8] =  cos_rX;
<a name="l03322"></a>03322 
<a name="l03323"></a>03323         <span class="comment">// printf(&quot;\n    -- Source location and direction for the following CT projection:\n&quot;);   // !!Verbose!! </span>
<a name="l03324"></a>03324         <span class="comment">// printf(&quot;                 angle between projections = %lf degrees\n&quot;, D_angle*RAD2DEG);</span>
<a name="l03325"></a>03325         <span class="comment">// printf(&quot;                             current angle = %lf degrees\n&quot;, current_angle*RAD2DEG);</span>
<a name="l03326"></a>03326         <span class="comment">// printf(&quot;                   new focal spot position = (%f, %f, %f)\n&quot;, source_data[i].position.x, source_data[i].position.y, source_data[i].position.z);</span>
<a name="l03327"></a>03327         <span class="comment">// printf(&quot;                      new source direction = (%f, %f, %f)\n&quot;, source_data[i].direction.x, source_data[i].direction.y, source_data[i].direction.z);</span>
<a name="l03328"></a>03328         <span class="comment">// printf(&quot;                       new detector center = (%f, %f, %f)\n&quot;, detector_center.x, detector_center.y, detector_center.z);</span>
<a name="l03329"></a>03329         <span class="comment">// printf(&quot;           new detector low corner (at +Y) = (%f, %f, %f)\n&quot;, detector_data-&gt;corner_min_rotated_to_Y[i].x, detector_data-&gt;corner_min_rotated_to_Y[i].y, detector_data-&gt;corner_min_rotated_to_Y[i].z);</span>
<a name="l03330"></a>03330         <span class="comment">// printf(&quot;                        center of rotation = (%f, %f, %f)\n&quot;, center_rotation.x, center_rotation.y, center_rotation.z);</span>
<a name="l03331"></a>03331         <span class="comment">// printf(&quot;         detector width (X) and height (Z) = %f , %f cm\n&quot;, detector_data-&gt;width_X, detector_data-&gt;height_Z);</span>
<a name="l03332"></a>03332         <span class="comment">// printf(&quot;            rotations to +Y around Z and X = %f , %f degrees\n&quot;, rotZ*RAD2DEG, rotX*RAD2DEG);</span>
<a name="l03333"></a>03333   }
<a name="l03334"></a>03334 }
<a name="l03335"></a>03335 
<a name="l03336"></a>03336 <span class="comment"></span>
<a name="l03337"></a>03337 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03338"></a>03338 <span class="comment"></span>
<a name="l03339"></a>03339 <span class="comment"></span>
<a name="l03340"></a>03340 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l03341"></a>03341 <span class="comment">//! Initialize the first seed of the pseudo-random number generator (PRNG) </span>
<a name="l03342"></a>03342 <span class="comment">//! RANECU to a position far away from the previous history (leap frog technique).</span>
<a name="l03343"></a>03343 <span class="comment">//! This function is equivalent to &quot;init_PRNG&quot; but only updates one of the seeds.</span>
<a name="l03344"></a>03344 <span class="comment">//!</span>
<a name="l03345"></a>03345 <span class="comment">//! Note that if we use the same seed number to initialize the 2 MLCGs of the PRNG</span>
<a name="l03346"></a>03346 <span class="comment">//! we can only warranty that the first MLCG will be uncorrelated for each value</span>
<a name="l03347"></a>03347 <span class="comment">//! generated by &quot;update_seed_PRNG&quot;. There is a tiny chance that the final PRNs will</span>
<a name="l03348"></a>03348 <span class="comment">//! be correlated because the leap frog on the first MLCG will probably go over the</span>
<a name="l03349"></a>03349 <span class="comment">//! repetition cycle of the MLCG, which is much smaller than the full RANECU. But any</span>
<a name="l03350"></a>03350 <span class="comment">//! correlataion is extremely unlikely. Function &quot;init_PRNG&quot; doesn&#39;t have this issue.</span>
<a name="l03351"></a>03351 <span class="comment">//!</span>
<a name="l03352"></a>03352 <span class="comment">//!       @param[in] batch_number   Elements to skip (eg, MPI thread_number).</span>
<a name="l03353"></a>03353 <span class="comment">//!       @param[in] total_histories   Histories to skip.</span>
<a name="l03354"></a>03354 <span class="comment">//!       @param[in,out] seed   Initial PRNG seeds; returns the updated seed.</span>
<a name="l03355"></a>03355 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l03356"></a><a class="code" href="MC-GPU__v1_83_8h.html#a6577c51b64b94b13c4e95df776275b5a">03356</a> <span class="comment"></span><span class="keyword">inline</span> <span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a6577c51b64b94b13c4e95df776275b5a" title="Initialize the first seed of the pseudo-random number generator (PRNG) RANECU to a position far away ...">update_seed_PRNG</a>(<span class="keywordtype">int</span> batch_number, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> total_histories, <span class="keywordtype">int</span>* seed)
<a name="l03357"></a>03357 {
<a name="l03358"></a>03358   <span class="keywordflow">if</span> (0==batch_number)
<a name="l03359"></a>03359     <span class="keywordflow">return</span>;
<a name="l03360"></a>03360     
<a name="l03361"></a>03361   <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">int</span> leap = total_histories * (batch_number * <a class="code" href="MC-GPU__kernel__v1_83_8cu.html#a3ab75c5bf72afd4a9b0255f61bd95704" title="Upper limit of the number of random values sampled in a single track.">LEAP_DISTANCE</a>);
<a name="l03362"></a>03362   <span class="keywordtype">int</span> y = 1;
<a name="l03363"></a>03363   <span class="keywordtype">int</span> z = <a class="code" href="MC-GPU__kernel__v1_83_8cu.html#afd7d5320ad79ad99c8d230f3a730141d" title="Multipliers and moduli for the two MLCG in RANECU.">a1_RANECU</a>;
<a name="l03364"></a>03364   <span class="comment">// -- Calculate the modulo power &#39;(a^leap)MOD(m)&#39; using a divide-and-conquer algorithm adapted to modulo arithmetic</span>
<a name="l03365"></a>03365   <span class="keywordflow">for</span>(;;)
<a name="l03366"></a>03366   {
<a name="l03367"></a>03367     <span class="comment">// (A2) Halve n, and store the integer part and the residue</span>
<a name="l03368"></a>03368     <span class="keywordflow">if</span> (0!=(leap&amp;01))  <span class="comment">// (bit-wise operation for MOD(leap,2), or leap%2 ==&gt; proceed if leap is an odd number)  Equivalent: t=(short)(leap%2);</span>
<a name="l03369"></a>03369     {
<a name="l03370"></a>03370       leap &gt;&gt;= 1;     <span class="comment">// Halve n moving the bits 1 position right. Equivalent to:  leap=(leap/2);  </span>
<a name="l03371"></a>03371       y = <a class="code" href="MC-GPU__kernel__v1_83_8cu.html#ae9061f4314474bf4415200a5cb115397" title="Calculate &quot;(a1*a2) MOD m&quot; with 32-bit integers and avoiding the possible overflow, using the Russian Peasant approach modulo m and the approximate factoring method, as described in: L&#39;Ecuyer and Cote, ACM Trans.">abMODm</a>(<a class="code" href="MC-GPU__kernel__v1_83_8cu.html#aa7d2cfcd3830e91736ca79f4bfdfb5d2">m1_RANECU</a>,z,y);      <span class="comment">// (A3) Multiply y by z:  y = [z*y] MOD m</span>
<a name="l03372"></a>03372       <span class="keywordflow">if</span> (0==leap) <span class="keywordflow">break</span>;         <span class="comment">// (A4) leap==0? ==&gt; finish</span>
<a name="l03373"></a>03373     }
<a name="l03374"></a>03374     <span class="keywordflow">else</span>           <span class="comment">// (leap is even)</span>
<a name="l03375"></a>03375     {
<a name="l03376"></a>03376       leap&gt;&gt;= 1;     <span class="comment">// Halve leap moving the bits 1 position right. Equivalent to:  leap=(leap/2);</span>
<a name="l03377"></a>03377     }
<a name="l03378"></a>03378     z = <a class="code" href="MC-GPU__kernel__v1_83_8cu.html#ae9061f4314474bf4415200a5cb115397" title="Calculate &quot;(a1*a2) MOD m&quot; with 32-bit integers and avoiding the possible overflow, using the Russian Peasant approach modulo m and the approximate factoring method, as described in: L&#39;Ecuyer and Cote, ACM Trans.">abMODm</a>(<a class="code" href="MC-GPU__kernel__v1_83_8cu.html#aa7d2cfcd3830e91736ca79f4bfdfb5d2">m1_RANECU</a>,z,z);        <span class="comment">// (A5) Square z:  z = [z*z] MOD m</span>
<a name="l03379"></a>03379   }
<a name="l03380"></a>03380   <span class="comment">// AjMODm1 = y;                 // Exponentiation finished:  AjMODm = expMOD = y = a^j</span>
<a name="l03381"></a>03381   <span class="comment">// -- Compute and display the seeds S(i+j), from the present seed S(i), using the previously calculated value of (a^j)MOD(m):</span>
<a name="l03382"></a>03382   <span class="comment">//         S(i+j) = [(a**j MOD m)*S(i)] MOD m</span>
<a name="l03383"></a>03383   <span class="comment">//         S_i = abMODm(m,S_i,AjMODm)</span>
<a name="l03384"></a>03384   *seed = <a class="code" href="MC-GPU__kernel__v1_83_8cu.html#ae9061f4314474bf4415200a5cb115397" title="Calculate &quot;(a1*a2) MOD m&quot; with 32-bit integers and avoiding the possible overflow, using the Russian Peasant approach modulo m and the approximate factoring method, as described in: L&#39;Ecuyer and Cote, ACM Trans.">abMODm</a>(<a class="code" href="MC-GPU__kernel__v1_83_8cu.html#aa7d2cfcd3830e91736ca79f4bfdfb5d2">m1_RANECU</a>, *seed, y);
<a name="l03385"></a>03385 }
<a name="l03386"></a>03386 
<a name="l03387"></a>03387 <span class="comment"></span>
<a name="l03388"></a>03388 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
<a name="l03389"></a>03389 <span class="comment"></span>
<a name="l03390"></a>03390 <span class="comment"></span>
<a name="l03391"></a>03391 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l03392"></a>03392 <span class="comment">//! Read the energy spectrum file and initialize the Walker aliasing sampling.</span>
<a name="l03393"></a>03393 <span class="comment">//!</span>
<a name="l03394"></a>03394 <span class="comment">//!       @param[in] file_name_espc   File containing the energy spectrum (lower energy value in each bin and its emission probability).</span>
<a name="l03395"></a>03395 <span class="comment">//!       @param[in,out] source_energy_data   Energy spectrum and other source data. The Walker alias and cutoffs are initialized in this function.</span>
<a name="l03396"></a>03396 <span class="comment">//!       @param[out] mean_energy_spectrum   Mean energy in the input x-ray energy spectrum.</span>
<a name="l03397"></a>03397 <span class="comment">////////////////////////////////////////////////////////////////////////////////</span>
<a name="l03398"></a><a class="code" href="MC-GPU__v1_83_8h.html#ac1d6abb26890276f529eed6b7ceafe89">03398</a> <span class="comment"></span><span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#ac1d6abb26890276f529eed6b7ceafe89" title="Read the energy spectrum file and initialize the Walker aliasing sampling.">init_energy_spectrum</a>(<span class="keywordtype">char</span>* file_name_espc, <span class="keyword">struct</span> <a class="code" href="structsource__energy__struct.html" title="Structure storing the source energy spectrum data to be sampled using the Walker aliasing algorithm...">source_energy_struct</a>* source_energy_data, <span class="keywordtype">float</span> *mean_energy_spectrum)
<a name="l03399"></a>03399 {
<a name="l03400"></a>03400   <span class="keywordtype">char</span> *new_line_ptr = NULL, new_line[250];    
<a name="l03401"></a>03401   <span class="keywordtype">float</span> lower_energy_bin, prob;
<a name="l03402"></a>03402   <span class="keywordtype">float</span> prob_espc_bin[<a class="code" href="MC-GPU__v1_83_8h.html#a5185f8c2c84b828fb1c961f51527709a" title="Maximum number of energy bins in the input x-ray energy spectrum.">MAX_ENERGY_BINS</a>];    <span class="comment">// The input probabilities of each energy bin will be discarded after Walker is initialized</span>
<a name="l03403"></a>03403 
<a name="l03404"></a>03404   <span class="comment">// -- Read spectrum from file:</span>
<a name="l03405"></a>03405   FILE* file_ptr = fopen(file_name_espc, <span class="stringliteral">&quot;r&quot;</span>);
<a name="l03406"></a>03406   <span class="keywordflow">if</span> (NULL==file_ptr)
<a name="l03407"></a>03407   {
<a name="l03408"></a>03408     printf(<span class="stringliteral">&quot;\n\n   !!init_energy_spectrum ERROR!! Error trying to read the energy spectrum input file \&quot;%s\&quot;.\n\n&quot;</span>, file_name_espc);
<a name="l03409"></a>03409     exit(-1);
<a name="l03410"></a>03410   }
<a name="l03411"></a>03411   
<a name="l03412"></a>03412   <span class="keywordtype">int</span> current_bin = -1;
<a name="l03413"></a>03413   <span class="keywordflow">do</span> 
<a name="l03414"></a>03414   {
<a name="l03415"></a>03415     current_bin++;  <span class="comment">// Update bin counter</span>
<a name="l03416"></a>03416     
<a name="l03417"></a>03417     <span class="keywordflow">if</span> (current_bin &gt;= <a class="code" href="MC-GPU__v1_83_8h.html#a5185f8c2c84b828fb1c961f51527709a" title="Maximum number of energy bins in the input x-ray energy spectrum.">MAX_ENERGY_BINS</a>)
<a name="l03418"></a>03418     {
<a name="l03419"></a>03419       printf(<span class="stringliteral">&quot;\n !!init_energy_spectrum ERROR!!: too many energy bins in the input spectrum. Increase the value of MAX_ENERGY_BINS=%d.\n&quot;</span>, <a class="code" href="MC-GPU__v1_83_8h.html#a5185f8c2c84b828fb1c961f51527709a" title="Maximum number of energy bins in the input x-ray energy spectrum.">MAX_ENERGY_BINS</a>);
<a name="l03420"></a>03420       printf(  <span class="stringliteral">&quot;            A negative probability marks the end of the spectrum.\n\n&quot;</span>);
<a name="l03421"></a>03421       exit(-1);
<a name="l03422"></a>03422     }
<a name="l03423"></a>03423 
<a name="l03424"></a>03424     new_line_ptr = <a class="code" href="MC-GPU__v1_83_8cu.html#aea168bca1466813e03b649931a4f670f" title="Read a line of text and trim initial blancks and trailing comments (#).">fgets_trimmed</a>(new_line, 250, file_ptr);   <span class="comment">// Read the following line of text skipping comments and extra spaces</span>
<a name="l03425"></a>03425     
<a name="l03426"></a>03426     <span class="keywordflow">if</span> (new_line_ptr==NULL)
<a name="l03427"></a>03427     {
<a name="l03428"></a>03428       printf(<span class="stringliteral">&quot;\n\n   !!init_energy_spectrum ERROR!! The input file for the x ray spectrum (%s) is not readable or incomplete (a negative probability marks the end of the spectrum).\n&quot;</span>, file_name_espc);
<a name="l03429"></a>03429       exit(-1);
<a name="l03430"></a>03430     }
<a name="l03431"></a>03431     
<a name="l03432"></a>03432     prob = -123456789.0f;  
<a name="l03433"></a>03433     
<a name="l03434"></a>03434     sscanf(new_line, <span class="stringliteral">&quot;%f %f&quot;</span>, &amp;lower_energy_bin, &amp;prob);     <span class="comment">// Extract the lowest energy in the bin and the corresponding emission probability from the line read </span>
<a name="l03435"></a>03435             
<a name="l03436"></a>03436     prob_espc_bin[current_bin]     = prob;
<a name="l03437"></a>03437     source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[current_bin] = lower_energy_bin;           
<a name="l03438"></a>03438     
<a name="l03439"></a>03439     <span class="keywordflow">if</span> (prob == -123456789.0f)
<a name="l03440"></a>03440     {
<a name="l03441"></a>03441       printf(<span class="stringliteral">&quot;\n !!init_energy_spectrum ERROR!!: invalid energy bin number %d?\n\n&quot;</span>, current_bin);
<a name="l03442"></a>03442       exit(-1);
<a name="l03443"></a>03443     }
<a name="l03444"></a>03444     <span class="keywordflow">else</span> <span class="keywordflow">if</span> (lower_energy_bin &lt; source_energy_data-&gt;espc[<a class="code" href="MC-GPU__v1_83_8h.html#a1b7b63b09e6ac252980467701bd80cb7" title="Preprocessor macro to calculate maximum and minimum values:">max_value</a>(current_bin-1,0)])    <span class="comment">// (Avoid a negative index using the macro &quot;max_value&quot; defined in the header file)</span>
<a name="l03445"></a>03445     {
<a name="l03446"></a>03446       printf(<span class="stringliteral">&quot;\n !!init_energy_spectrum ERROR!!: input energy bins with decreasing energy? espc(%d)=%f, espc(%d)=%f\n\n&quot;</span>, current_bin-1, source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[<a class="code" href="MC-GPU__v1_83_8h.html#a1b7b63b09e6ac252980467701bd80cb7" title="Preprocessor macro to calculate maximum and minimum values:">max_value</a>(current_bin-1,0)], current_bin, lower_energy_bin);
<a name="l03447"></a>03447       exit(-1);
<a name="l03448"></a>03448     }
<a name="l03449"></a>03449     
<a name="l03450"></a>03450   } 
<a name="l03451"></a>03451   <span class="keywordflow">while</span> (prob &gt; -1.0e-11f);     <span class="comment">// A negative probability marks the end of the spectrum</span>
<a name="l03452"></a>03452 
<a name="l03453"></a>03453 
<a name="l03454"></a>03454   <span class="comment">// Store the number of bins read from the input energy spectrum file:</span>
<a name="l03455"></a>03455   source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a> = current_bin;
<a name="l03456"></a>03456 
<a name="l03457"></a>03457 
<a name="l03458"></a>03458   <span class="comment">// Init the remaining bins (which will not be used) with the last energy read (will be assumed as the highest energy in the last bin) and 0 probability of emission.</span>
<a name="l03459"></a>03459   <span class="keyword">register</span> <span class="keywordtype">int</span> i;
<a name="l03460"></a>03460   <span class="keywordflow">for</span> (i=current_bin; i&lt;<a class="code" href="MC-GPU__v1_83_8h.html#a5185f8c2c84b828fb1c961f51527709a" title="Maximum number of energy bins in the input x-ray energy spectrum.">MAX_ENERGY_BINS</a>; i++)
<a name="l03461"></a>03461   {
<a name="l03462"></a>03462     source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[i] = lower_energy_bin;
<a name="l03463"></a>03463     prob_espc_bin[i]     = 0.0f;
<a name="l03464"></a>03464   }
<a name="l03465"></a>03465 
<a name="l03466"></a>03466 
<a name="l03467"></a>03467   <span class="comment">// Compute the mean energy in the spectrum, taking into account the energy and prob of each bin:</span>
<a name="l03468"></a>03468   <span class="keywordtype">float</span> all_energy = 0.0f;
<a name="l03469"></a>03469   <span class="keywordtype">float</span> all_prob = 0.0f;
<a name="l03470"></a>03470   <span class="keywordflow">for</span>(i=0; i&lt;source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a>; i++)
<a name="l03471"></a>03471   {
<a name="l03472"></a>03472     all_energy += 0.5f*(source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[i]+source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a66db20f7a335eb91023e0d1191730b42">espc</a>[i+1])*prob_espc_bin[i];
<a name="l03473"></a>03473     all_prob   += prob_espc_bin[i];
<a name="l03474"></a>03474   }  
<a name="l03475"></a>03475   *mean_energy_spectrum = all_energy/all_prob;
<a name="l03476"></a>03476   
<a name="l03477"></a>03477           
<a name="l03478"></a>03478 <span class="comment">// -- Init the Walker aliasing sampling method (as it is done in PENELOPE):</span>
<a name="l03479"></a>03479   <a class="code" href="MC-GPU__v1_83_8cu.html#ac2eacae33c0451f6a23fdf1101c6ca11" title="Initialisation of Walker&#39;s aliasing algorithm for random sampling from discrete probability distribut...">IRND0</a>(prob_espc_bin, source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a39635503e804b265f76d3a0f7edb2a9f">espc_cutoff</a>, source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a8504b5d435cbf4feef2af89c0fd3f829">espc_alias</a>, source_energy_data-&gt;<a class="code" href="structsource__energy__struct.html#a559e19adbc83b2aea219281e14f82bb1">num_bins_espc</a>);   <span class="comment">//!!Walker!! Calling PENELOPE&#39;s function to init the Walker method</span>
<a name="l03480"></a>03480 <span class="comment"></span>       
<a name="l03481"></a>03481 <span class="comment">// !!Verbose!! Test sampling</span>
<a name="l03482"></a>03482 <span class="comment">// Sampling the x ray energy using the Walker aliasing algorithm from PENELOPE:</span>
<a name="l03483"></a>03483 <span class="comment">// int sampled_bin = seeki_walker(source_energy_data-&gt;espc_cutoff, source_energy_data-&gt;espc_alias, 0.5, source_energy_data-&gt;num_bins_espc);</span>
<a name="l03484"></a>03484 <span class="comment">// float e = source_energy_data-&gt;espc[sampled_bin] + ranecu(seed) * (source_energy_data-&gt;espc[sampled_bin+1] - source_energy_data-&gt;espc[sampled_bin]);    // Linear interpolation of the final energy within the sampled energy bin</span>
<a name="l03485"></a>03485 <span class="comment">// printf(&quot;\n\n !!Walker!! Energy center bin %d = %f keV\n&quot;, sampled_bin, 0.001f*e);</span>
<a name="l03486"></a>03486   
<a name="l03487"></a>03487 }       
<a name="l03488"></a>03488 
<a name="l03489"></a>03489 
<a name="l03490"></a>03490 
<a name="l03491"></a>03491       
<a name="l03492"></a>03492 <span class="comment">//********************************************************************</span><span class="comment"></span>
<a name="l03493"></a>03493 <span class="comment">//!    Finds the interval (x(i),x(i+1)] containing the input value    </span>
<a name="l03494"></a>03494 <span class="comment">//!    using Walker&#39;s aliasing method.                                </span>
<a name="l03495"></a>03495 <span class="comment">//!                                                                   </span>
<a name="l03496"></a>03496 <span class="comment">//!    Input:                                                         </span>
<a name="l03497"></a>03497 <span class="comment">//!      cutoff(1..n) -&gt; interval cutoff values for the Walker method </span>
<a name="l03498"></a>03498 <span class="comment">//!      cutoff(1..n) -&gt; alias for the upper part of each interval    </span>
<a name="l03499"></a>03499 <span class="comment">//!      randno       -&gt; point to be located                          </span>
<a name="l03500"></a>03500 <span class="comment">//!      n            -&gt; no. of data points                           </span>
<a name="l03501"></a>03501 <span class="comment">//!    Output:                                                        </span>
<a name="l03502"></a>03502 <span class="comment">//!      index i of the semiopen interval where randno lies           </span>
<a name="l03503"></a>03503 <span class="comment">//!    Comments:                                                      </span>
<a name="l03504"></a>03504 <span class="comment">//!      -&gt; The cutoff and alias values have to be previously         </span>
<a name="l03505"></a>03505 <span class="comment">//!         initialised calling the penelope subroutine IRND0.        </span>
<a name="l03506"></a>03506 <span class="comment">//!                                                                   </span>
<a name="l03507"></a>03507 <span class="comment">//!                                                                   </span>
<a name="l03508"></a>03508 <span class="comment">//!    Algorithm implementation based on the PENELOPE code developed   </span>
<a name="l03509"></a>03509 <span class="comment">//!    by Francesc Salvat at the University of Barcelona. For more     </span>
<a name="l03510"></a>03510 <span class="comment">//!    info: www.oecd-nea.org/science/pubs/2009/nea6416-penelope.pdf  </span>
<a name="l03511"></a>03511 <span class="comment">//!                                                                   </span>
<a name="l03512"></a>03512 <span class="comment"></span><span class="comment">//CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC</span>
<a name="l03513"></a>03513 <span class="comment">//C  PENELOPE/PENGEOM (version 2006)                                     C</span>
<a name="l03514"></a>03514 <span class="comment">//C  Copyright (c) 2001-2006                                             C</span>
<a name="l03515"></a>03515 <span class="comment">//C  Universitat de Barcelona                                            C</span>
<a name="l03516"></a>03516 <span class="comment">//C                                                                      C</span>
<a name="l03517"></a>03517 <span class="comment">//C  Permission to use, copy, modify, distribute and sell this software  C</span>
<a name="l03518"></a>03518 <span class="comment">//C  and its documentation for any purpose is hereby granted without     C</span>
<a name="l03519"></a>03519 <span class="comment">//C  fee, provided that the above copyright notice appears in all        C</span>
<a name="l03520"></a>03520 <span class="comment">//C  copies and that both that copyright notice and this permission      C</span>
<a name="l03521"></a>03521 <span class="comment">//C  notice appear in all supporting documentation. The Universitat de   C</span>
<a name="l03522"></a>03522 <span class="comment">//C  Barcelona makes no representations about the suitability of this    C</span>
<a name="l03523"></a>03523 <span class="comment">//C  software for any purpose. It is provided &quot;as is&quot; without express    C</span>
<a name="l03524"></a>03524 <span class="comment">//C  or implied warranty.                                                C</span>
<a name="l03525"></a>03525 <span class="comment">//CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC</span>
<a name="l03526"></a><a class="code" href="MC-GPU__v1_83_8h.html#a0b858ab906084d4426bfe5b522074e2c">03526</a> <span class="keyword">inline</span> <span class="keywordtype">int</span> <a class="code" href="MC-GPU__v1_83_8cu.html#a0b858ab906084d4426bfe5b522074e2c" title="Finds the interval (x(i),x(i+1)] containing the input value using Walker&#39;s aliasing method...">seeki_walker</a>(<span class="keywordtype">float</span> *cutoff, <span class="keywordtype">short</span> <span class="keywordtype">int</span> *alias, <span class="keywordtype">float</span> randno, <span class="keywordtype">int</span> n)
<a name="l03527"></a>03527 {
<a name="l03528"></a>03528    <span class="keywordtype">float</span> RN = randno * n;                         <span class="comment">// Find initial interval (array starting at 0):   </span>
<a name="l03529"></a>03529    <span class="keywordtype">int</span> int_part = (int)(RN);                      <span class="comment">//   -- Integer part</span>
<a name="l03530"></a>03530    <span class="keywordtype">float</span> fraction_part = RN - ((float)int_part);  <span class="comment">//   -- Fractional part</span>
<a name="l03531"></a>03531 
<a name="l03532"></a>03532    <span class="keywordflow">if</span> (fraction_part &lt; cutoff[int_part])          <span class="comment">// Check if we are in the aliased part</span>
<a name="l03533"></a>03533       <span class="keywordflow">return</span> int_part;                            <span class="comment">// Below the cutoff: return current value</span>
<a name="l03534"></a>03534    <span class="keywordflow">else</span>
<a name="l03535"></a>03535       <span class="keywordflow">return</span> (<span class="keywordtype">int</span>)alias[int_part];                <span class="comment">// Above the cutoff: return alias</span>
<a name="l03536"></a>03536 }     
<a name="l03537"></a>03537 
<a name="l03538"></a>03538 
<a name="l03539"></a>03539 
<a name="l03540"></a>03540 
<a name="l03541"></a>03541 <span class="comment">//****************************************************************** *</span>
<a name="l03542"></a>03542 <span class="comment">//*                    SUBROUTINE IRND0                              *</span>
<a name="l03543"></a>03543 <span class="comment">//********************************************************************</span>
<a name="l03544"></a>03544 <span class="comment">//*                                                                   </span><span class="comment"></span>
<a name="l03545"></a>03545 <span class="comment">//!  Initialisation of Walker&#39;s aliasing algorithm for random         </span>
<a name="l03546"></a>03546 <span class="comment">//!  sampling from discrete probability distributions.                </span>
<a name="l03547"></a>03547 <span class="comment">//!                                                                   </span>
<a name="l03548"></a>03548 <span class="comment">//! Input arguments:                                                  </span>
<a name="l03549"></a>03549 <span class="comment">//!   N ........ number of different values of the random variable.   </span>
<a name="l03550"></a>03550 <span class="comment">//!   W(1:N) ... corresponding point probabilities (not necessarily   </span>
<a name="l03551"></a>03551 <span class="comment">//!              normalised to unity).                                </span>
<a name="l03552"></a>03552 <span class="comment">//! Output arguments:                                                 </span>
<a name="l03553"></a>03553 <span class="comment">//!   F(1:N) ... cutoff values.                                       </span>
<a name="l03554"></a>03554 <span class="comment">//!   K(1:N) ... alias values.                                        </span>
<a name="l03555"></a>03555 <span class="comment">//!                                                                   </span>
<a name="l03556"></a>03556 <span class="comment">//!                                                                   </span>
<a name="l03557"></a>03557 <span class="comment">//!  This subroutine is part of the PENELOPE 2006 code developed      </span>
<a name="l03558"></a>03558 <span class="comment">//!  by Francesc Salvat at the University of Barcelona. For more       </span>
<a name="l03559"></a>03559 <span class="comment">//!  info: www.oecd-nea.org/science/pubs/2009/nea6416-penelope.pdf    </span>
<a name="l03560"></a>03560 <span class="comment"></span><span class="comment">//*                                                                   </span>
<a name="l03561"></a>03561 <span class="comment">//CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC</span>
<a name="l03562"></a>03562 <span class="comment">//C  PENELOPE/PENGEOM (version 2006)                                     C</span>
<a name="l03563"></a>03563 <span class="comment">//C  Copyright (c) 2001-2006                                             C</span>
<a name="l03564"></a>03564 <span class="comment">//C  Universitat de Barcelona                                            C</span>
<a name="l03565"></a>03565 <span class="comment">//C                                                                      C</span>
<a name="l03566"></a>03566 <span class="comment">//C  Permission to use, copy, modify, distribute and sell this software  C</span>
<a name="l03567"></a>03567 <span class="comment">//C  and its documentation for any purpose is hereby granted without     C</span>
<a name="l03568"></a>03568 <span class="comment">//C  fee, provided that the above copyright notice appears in all        C</span>
<a name="l03569"></a>03569 <span class="comment">//C  copies and that both that copyright notice and this permission      C</span>
<a name="l03570"></a>03570 <span class="comment">//C  notice appear in all supporting documentation. The Universitat de   C</span>
<a name="l03571"></a>03571 <span class="comment">//C  Barcelona makes no representations about the suitability of this    C</span>
<a name="l03572"></a>03572 <span class="comment">//C  software for any purpose. It is provided &quot;as is&quot; without express    C</span>
<a name="l03573"></a>03573 <span class="comment">//C  or implied warranty.                                                C</span>
<a name="l03574"></a>03574 <span class="comment">//CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC</span>
<a name="l03575"></a><a class="code" href="MC-GPU__v1_83_8h.html#ac2eacae33c0451f6a23fdf1101c6ca11">03575</a> <span class="keywordtype">void</span> <a class="code" href="MC-GPU__v1_83_8cu.html#ac2eacae33c0451f6a23fdf1101c6ca11" title="Initialisation of Walker&#39;s aliasing algorithm for random sampling from discrete probability distribut...">IRND0</a>(<span class="keywordtype">float</span> *W, <span class="keywordtype">float</span> *F, <span class="keywordtype">short</span> <span class="keywordtype">int</span> *K, <span class="keywordtype">int</span> N)
<a name="l03576"></a>03576 {
<a name="l03577"></a>03577    <span class="keyword">register</span> <span class="keywordtype">int</span> I;
<a name="l03578"></a>03578   
<a name="l03579"></a>03579    <span class="comment">//  ****  Renormalisation.</span>
<a name="l03580"></a>03580    <span class="keywordtype">double</span> WS=0.0;
<a name="l03581"></a>03581    <span class="keywordflow">for</span> (I=0; I&lt;N; I++)
<a name="l03582"></a>03582    {   
<a name="l03583"></a>03583       <span class="keywordflow">if</span>(W[I] &lt; 0.0f) 
<a name="l03584"></a>03584       {
<a name="l03585"></a>03585          printf(<span class="stringliteral">&quot;\n\n !!ERROR!! IRND0: Walker sampling initialization. Negative point probability? W(%d)=%f\n\n&quot;</span>, I, W[I]);
<a name="l03586"></a>03586          exit(-1);
<a name="l03587"></a>03587       }
<a name="l03588"></a>03588       WS = WS + W[I];
<a name="l03589"></a>03589    }
<a name="l03590"></a>03590    WS = ((double)N) / WS; 
<a name="l03591"></a>03591   
<a name="l03592"></a>03592    <span class="keywordflow">for</span> (I=0; I&lt;N; I++)
<a name="l03593"></a>03593    {
<a name="l03594"></a>03594       K[I] = I;
<a name="l03595"></a>03595       F[I] = W[I] * WS;
<a name="l03596"></a>03596    }
<a name="l03597"></a>03597     
<a name="l03598"></a>03598    <span class="keywordflow">if</span> (N==1) 
<a name="l03599"></a>03599       <span class="keywordflow">return</span>;
<a name="l03600"></a>03600      
<a name="l03601"></a>03601    <span class="comment">//  ****  Cutoff and alias values.</span>
<a name="l03602"></a>03602    <span class="keywordtype">float</span> HLOW, HIGH;
<a name="l03603"></a>03603    <span class="keywordtype">int</span>   ILOW, IHIGH, J;
<a name="l03604"></a>03604    <span class="keywordflow">for</span> (I=0; I&lt;N-1; I++)
<a name="l03605"></a>03605    {
<a name="l03606"></a>03606       HLOW = 1.0f;
<a name="l03607"></a>03607       HIGH = 1.0f;
<a name="l03608"></a>03608       ILOW = -1;
<a name="l03609"></a>03609       IHIGH= -1;
<a name="l03610"></a>03610       <span class="keywordflow">for</span> (J=0; J&lt;N; J++)
<a name="l03611"></a>03611       {
<a name="l03612"></a>03612          <span class="keywordflow">if</span>(K[J]==J)
<a name="l03613"></a>03613          {
<a name="l03614"></a>03614             <span class="keywordflow">if</span>(F[J]&lt;HLOW)
<a name="l03615"></a>03615             {
<a name="l03616"></a>03616                HLOW = F[J];
<a name="l03617"></a>03617                ILOW = J;
<a name="l03618"></a>03618             }
<a name="l03619"></a>03619             <span class="keywordflow">else</span> <span class="keywordflow">if</span>(F[J]&gt;HIGH)
<a name="l03620"></a>03620             {
<a name="l03621"></a>03621                HIGH  = F[J];
<a name="l03622"></a>03622                IHIGH = J;
<a name="l03623"></a>03623             }
<a name="l03624"></a>03624          }
<a name="l03625"></a>03625       }
<a name="l03626"></a>03626       
<a name="l03627"></a>03627       <span class="keywordflow">if</span>((ILOW==-1) || (IHIGH==-1)) 
<a name="l03628"></a>03628         <span class="keywordflow">return</span>;
<a name="l03629"></a>03629 
<a name="l03630"></a>03630       K[ILOW] = IHIGH;
<a name="l03631"></a>03631       F[IHIGH]= HIGH + HLOW - 1.0f;
<a name="l03632"></a>03632    }
<a name="l03633"></a>03633    <span class="keywordflow">return</span>;
<a name="l03634"></a>03634 }
<a name="l03635"></a>03635 
<a name="l03636"></a>03636 
<a name="l03637"></a>03637 <span class="comment"></span>
<a name="l03638"></a>03638 <span class="comment">///////////////////////////////////////////////////////////////////////////////</span>
</pre></div></div><!-- contents -->
</div>
  <div id="nav-path" class="navpath">
    <ul>
      <li class="navelem"><a class="el" href="MC-GPU__v1_83_8cu.html">MC-GPU_v1.3.cu</a>      </li>

    <li class="footer">Generated on Mon Dec 17 2012 13:26:39 for MC-GPU by
    <a href="http://www.doxygen.org/index.html">
    <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.6.1 </li>
   </ul>
 </div>


</body>
</html>
